Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): add decision endpoint (#456)

nachiketb-nvidia210766446028
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 65c5083b6..75d1b1fd1 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -128,6 +128,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_
| `POST` | `/v1/chat/completions` | OpenAI Chat Completions |
| `POST` | `/v1/messages` | Anthropic Messages |
| `POST` | `/v1/responses` | OpenAI Responses |
+| `POST` | `/v1/decision` | Resolve selected and fallback targets without a post-routing answer call |
| `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target |
| `GET` | `/v1/models` | Routes served by this deployment |
| `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats |
@@ -139,6 +140,14 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":
routes through the `[routes.general]` entry above. Any of the three request formats can address any
route, and the server translates between them.
+`POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where the
+nested request names the route in `model`. It executes required classifier or judge calls, then
+returns the selected target and ordered fallbacks with their model, format, base URL, and
+`extra_body`. It does not make a post-routing answer call. Routing-time calls still execute, and
+response-dependent algorithms such as escalation and advisor routing may produce an answer while
+deciding. When they do, the endpoint includes the buffered answer as `response`, encoded in
+`input_format`; otherwise the field is omitted.
+
For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic
target and summarizes its score, confidence, and input-dimension histograms. These values reset
with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative.
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index a78695cd5..420ac5e2b 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -20,7 +20,7 @@ use switchyard_llm_client::{
Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
TranslatingLlmClient,
};
-use switchyard_protocol::{ModelId, RoutedLlmClient};
+use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat};
use crate::{
CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState,
@@ -44,22 +44,31 @@ pub fn load_server_state(path: impl AsRef<Path>) -> ServerResult<ServerState> {
}
fn server_state_from_toml(toml: &str) -> ServerResult<ServerState> {
- let config: ServerConfig = toml::from_str(toml)
- .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?;
- config.build()
+ let config: Arc<ServerConfig> = Arc::new(
+ toml::from_str(toml)
+ .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?,
+ );
+ let state = config.build()?;
+ Ok(state.with_config(config))
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
-struct ServerConfig {
+pub(crate) struct ServerConfig {
schema_version: u32,
#[serde(default)]
- llm_clients: BTreeMap<String, LlmClientConfig>,
- targets: BTreeMap<String, TargetConfig>,
+ pub(crate) llm_clients: BTreeMap<String, LlmClientConfig>,
+ pub(crate) targets: BTreeMap<String, TargetConfig>,
routes: BTreeMap<String, RouteConfig>,
}
impl ServerConfig {
+ pub(crate) fn routing_target_names(&self, route_name: &str) -> Option<Vec<&str>> {
+ self.routes
+ .get(route_name)
+ .map(RouteConfig::routing_target_names)
+ }
+
fn build(&self) -> ServerResult<ServerState> {
if self.schema_version != SUPPORTED_SCHEMA_VERSION {
return Err(ServerError::new(format!(
@@ -94,6 +103,11 @@ impl ServerConfig {
for (route_name, config) in &self.routes {
validate_value("route name", route_name)?;
validate_value(&format!("route {route_name} id"), config.id())?;
+ for target_name in config.callable_target_names() {
+ self.targets.get(target_name).ok_or_else(|| {
+ ServerError::new(format!("route references unknown target {target_name}"))
+ })?;
+ }
let capabilities = config.capabilities();
if capabilities.context_window == Some(0) {
return Err(ServerError::new(format!(
@@ -110,6 +124,7 @@ impl ServerConfig {
caller_auth,
capabilities,
count_tokens_target,
+ Some(route_name.clone()),
));
}
ServerState::new_with_capabilities(routes)
@@ -246,9 +261,9 @@ fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize {
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
-struct LlmClientConfig {
- format: ClientFormat,
- base_url: String,
+pub(crate) struct LlmClientConfig {
+ pub(crate) format: ClientFormat,
+ pub(crate) base_url: String,
api_key_env: Option<String>,
#[serde(default)]
forward_auth: bool,
@@ -260,15 +275,15 @@ struct LlmClientConfig {
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
-struct TargetConfig {
- id: ModelId,
- llm_client: String,
+pub(crate) struct TargetConfig {
+ pub(crate) id: ModelId,
+ pub(crate) llm_client: String,
#[serde(default)]
- extra_body: BTreeMap<String, Value>,
+ pub(crate) extra_body: BTreeMap<String, Value>,
}
#[derive(Clone, Copy, Debug, Deserialize)]
-enum ClientFormat {
+pub(crate) enum ClientFormat {
#[serde(rename = "openai_chat")]
OpenAiChat,
#[serde(rename = "openai_responses")]
@@ -278,6 +293,14 @@ enum ClientFormat {
}
impl ClientFormat {
+ pub(crate) const fn wire_format(self) -> WireFormat {
+ match self {
+ Self::OpenAiChat => WireFormat::OpenAiChat,
+ Self::OpenAiResponses => WireFormat::OpenAiResponses,
+ Self::AnthropicMessages => WireFormat::AnthropicMessages,
+ }
+ }
+
const fn caller_auth_kind(self) -> CallerAuthKind {
match self {
Self::AnthropicMessages => CallerAuthKind::Anthropic,
@@ -1493,9 +1516,10 @@ classifier_magic = true
];
for (toml, expected) in cases {
+ let error = error_message(&toml);
assert!(
- error_message(&toml).contains(expected),
- "expected error containing {expected}"
+ error.contains(expected),
+ "expected error containing {expected}, got {error}"
);
}
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index e302e29dd..483c1f6a6 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -32,9 +32,9 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Extension, Json, Router};
use axum_server::tls_rustls::RustlsConfig;
-use libsy::{Algorithm, LibsyError};
+use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive};
use parking_lot::Mutex;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};
use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};
@@ -42,8 +42,9 @@ use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
use tracing::{Instrument, Level};
-use switchyard_translation::{WireFormat, decode_request};
+use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response};
+use crate::config::ServerConfig;
use crate::response::into_http_response;
use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env};
@@ -112,6 +113,32 @@ struct RouteEntry {
caller_auth: Option<CallerAuthKind>,
capabilities: ModelCapabilities,
count_tokens_target: Option<CountTokensTarget>,
+ config_name: Option<String>,
+}
+
+/// Decision result using the same selected/fallback order as libsy.
+#[derive(Serialize)]
+struct DecisionResponse<'a> {
+ selected: DecisionTargetResponse<'a>,
+ fallbacks: Vec<DecisionTargetResponse<'a>>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ response: Option<Value>,
+}
+
+/// Borrowed response view over one target's existing configuration.
+#[derive(Serialize)]
+struct DecisionTargetResponse<'a> {
+ target: &'a str,
+ model: &'a ModelId,
+ llm_client: DecisionLlmClientResponse<'a>,
+ extra_body: &'a BTreeMap<String, Value>,
+}
+
+/// Non-secret client settings needed to call a selected model.
+#[derive(Serialize)]
+struct DecisionLlmClientResponse<'a> {
+ format: WireFormat,
+ base_url: &'a str,
}
/// Caller credential family required by forwarded-auth backends.
@@ -158,6 +185,7 @@ impl CountTokensTarget {
#[derive(Clone)]
pub struct ServerState {
routes: Arc<BTreeMap<ModelId, RouteEntry>>,
+ config: Option<Arc<ServerConfig>>,
metrics: prometheus::Registry,
stats: StatsAccumulator,
routing_log: Option<SharedRoutingLog>,
@@ -212,6 +240,7 @@ impl ServerState {
None,
ModelCapabilities::default(),
None,
+ None,
)
}))
}
@@ -225,12 +254,20 @@ impl ServerState {
Option<CallerAuthKind>,
ModelCapabilities,
Option<CountTokensTarget>,
+ Option<String>,
),
>,
) -> ServerResult<Self> {
let mut entries = BTreeMap::new();
- for (model, algorithm, target_clients, caller_auth, capabilities, count_tokens_target) in
- routes
+ for (
+ model,
+ algorithm,
+ target_clients,
+ caller_auth,
+ capabilities,
+ count_tokens_target,
+ config_name,
+ ) in routes
{
let model = ModelId::from(model.trim());
if model.is_empty() {
@@ -242,6 +279,7 @@ impl ServerState {
caller_auth,
capabilities,
count_tokens_target,
+ config_name,
};
if entries.insert(model.clone(), entry).is_some() {
return Err(ServerError::new(format!("duplicate route model {model}")));
@@ -257,6 +295,7 @@ impl ServerState {
);
Ok(Self {
routes: Arc::new(entries),
+ config: None,
metrics,
stats,
routing_log: None,
@@ -264,6 +303,12 @@ impl ServerState {
})
}
+ /// Retains the validated configuration used to describe decision results.
+ fn with_config(mut self, config: Arc<ServerConfig>) -> Self {
+ self.config = Some(config);
+ self
+ }
+
/// Enables durable per-request routing records at `path`.
pub fn with_routing_log(mut self, path: impl Into<PathBuf>) -> ServerResult<Self> {
self.routing_log = Some(SharedRoutingLog::new(path.into())?);
@@ -285,6 +330,42 @@ impl ServerState {
fn route_for_model(&self, model: &str) -> Option<&RouteEntry> {
self.routes.get(model)
}
+
+ /// Resolves the routing outcome through the target names configured for its route.
+ fn decision_response<'a>(
+ &'a self,
+ route: &'a RouteEntry,
+ outcome: &RoutingOutcome,
+ response: Option<Value>,
+ ) -> Option<DecisionResponse<'a>> {
+ let config = self.config.as_ref()?;
+ let target_names = config.routing_target_names(route.config_name.as_ref()?)?;
+ let resolve = |model: &ModelId| {
+ let (target_name, target) = target_names
+ .iter()
+ .filter_map(|name| config.targets.get_key_value(*name))
+ .find(|(_, target)| target.id == *model)?;
+ let client = config.llm_clients.get(&target.llm_client)?;
+ Some(DecisionTargetResponse {
+ target: target_name,
+ model: &target.id,
+ llm_client: DecisionLlmClientResponse {
+ format: client.format.wire_format(),
+ base_url: &client.base_url,
+ },
+ extra_body: &target.extra_body,
+ })
+ };
+ Some(DecisionResponse {
+ selected: resolve(&outcome.selected_model_id)?,
+ fallbacks: outcome
+ .fallback_models
+ .iter()
+ .map(resolve)
+ .collect::<Option<Vec<_>>>()?,
+ response,
+ })
+ }
}
/// Runtime options shared by server entry points.
@@ -503,6 +584,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router {
.route("/v1/chat/completions", post(openai_chat_completions))
.route("/v1/messages", post(anthropic_messages))
.route("/v1/responses", post(openai_responses))
+ .route("/v1/decision", post(decision))
.route("/v1/messages/count_tokens", post(anthropic_count_tokens))
.route("/v1/models", get(models))
.route("/v1/stats", get(get_stats))
@@ -564,6 +646,92 @@ async fn openai_responses(
handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await
}
+/// One provider request submitted for a routing decision without an answer-model call.
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct DecisionEndpointRequest {
+ input_format: WireFormat,
+ request: Value,
+}
+
+/// Selects a target while still allowing the algorithm's classifier and judge calls.
+async fn decision(
+ State(state): State<ServerState>,
+ headers: HeaderMap,
+ body: std::result::Result<Json<DecisionEndpointRequest>, JsonRejection>,
+) -> Response {
+ let body = match body {
+ Ok(Json(body)) if body.request.is_object() => body,
+ Ok(_) => {
+ return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object");
+ }
+ Err(error) => {
+ return invalid_body_error(
+ error.status(),
+ format!("Request body must be valid JSON: {error}"),
+ );
+ }
+ };
+ let input_format = body.input_format;
+ let (route, request) = match resolve_route(
+ &state,
+ metadata_from_headers(headers),
+ body.request,
+ input_format,
+ ) {
+ Ok(resolved) => resolved,
+ Err(response) => return response,
+ };
+ let mut outcome = match run_decision_only(route, request).await {
+ Ok(outcome) => outcome,
+ Err(error) => return algorithm_error(error),
+ };
+ let response = match outcome.response.take() {
+ Some(response) => {
+ let aggregate = match response.llm_response.into_agg().await {
+ Ok(aggregate) => aggregate,
+ Err(error) => return client_error(&error),
+ };
+ match encode_aggregated_response(
+ &aggregate,
+ input_format,
+ Some(outcome.selected_model_id.as_str()),
+ ) {
+ Ok(response) => Some(response),
+ Err(error) => return server_error(error.to_string()),
+ }
+ }
+ None => None,
+ };
+ match state.decision_response(route, &outcome, response) {
+ Some(response) => Json(response).into_response(),
+ None => {
+ server_error("routing outcome contains a model with no callable target configuration")
+ }
+ }
+}
+
+/// Completes routing-time calls and returns the outcome without serving its answer target.
+async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result<RoutingOutcome> {
+ drive(Arc::clone(&route.algorithm), request, |call| {
+ serve_decision_dependency(route, call)
+ })
+ .await
+}
+
+/// Serves a classifier or judge call that the decision depends on.
+async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> {
+ let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
+ let result = match route.target_clients.route(&model) {
+ Ok(client) => client
+ .call(call.request.clone())
+ .await
+ .map_err(|source| LibsyError::client_call(model, source)),
+ Err(source) => Err(LibsyError::client_call(model, source)),
+ };
+ call.respond(result)
+}
+
/// Anthropic token counting against the route's explicitly configured target.
async fn anthropic_count_tokens(
State(state): State<ServerState>,
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 0a97f90e3..21b1e46da 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -956,6 +956,132 @@ base_threshold = 0.5
Ok(())
}
+/// Decision-only routing returns callable metadata and preserves any answer produced while routing.
+#[tokio::test]
+async fn decision_returns_callable_target_and_routing_answer() -> TestResult {
+ let judge_upstream = MockUpstream::start().await?;
+ let model_upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.judge_provider]
+format = "openai_chat"
+base_url = "{judge_url}"
+
+[llm_clients.model_provider]
+format = "openai_chat"
+base_url = "{model_url}"
+
+[targets.judge]
+id = "model/classifier"
+llm_client = "judge_provider"
+
+[targets.quality]
+id = "model/strong"
+llm_client = "model_provider"
+
+[targets.economy]
+id = "model/weak"
+llm_client = "model_provider"
+extra_body = {{ service_tier = "priority" }}
+
+[routes.classify]
+id = "switchyard/classify"
+type = "llm_classifier"
+classifier_target = "judge"
+strong_target = "quality"
+weak_target = "economy"
+base_threshold = 0.5
+
+[routes.escalation]
+id = "switchyard/escalation"
+type = "llm_classifier"
+mode = "escalation"
+classifier_target = "judge"
+strong_target = "quality"
+weak_target = "economy"
+escalation = {{ confirmations = 1 }}
+"#,
+ judge_url = judge_upstream.base_url,
+ model_url = model_upstream.base_url,
+ ))?;
+ let app = build_switchyard_router(state);
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/decision",
+ Some(json!({
+ "input_format": "openai_chat",
+ "request": {
+ "model": "switchyard/classify",
+ "messages": [{"role": "user", "content": "bounded task"}]
+ }
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ assert_eq!(
+ response.json()?,
+ json!({
+ "selected": {
+ "target": "economy",
+ "model": "model/weak",
+ "llm_client": {
+ "format": "openai_chat",
+ "base_url": model_upstream.base_url,
+ },
+ "extra_body": {"service_tier": "priority"},
+ },
+ "fallbacks": [{
+ "target": "quality",
+ "model": "model/strong",
+ "llm_client": {
+ "format": "openai_chat",
+ "base_url": model_upstream.base_url,
+ },
+ "extra_body": {},
+ }],
+ })
+ );
+ assert_eq!(
+ judge_upstream.models().await,
+ vec!["model/classifier".to_string()]
+ );
+ assert!(model_upstream.models().await.is_empty());
+
+ judge_upstream.calls.lock().await.clear();
+ model_upstream.calls.lock().await.clear();
+ let response = send(
+ &app,
+ "POST",
+ "/v1/decision",
+ Some(json!({
+ "input_format": "openai_chat",
+ "request": {
+ "model": "switchyard/escalation",
+ "messages": [{"role": "user", "content": "bounded task"}]
+ }
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ let response = response.json()?;
+ assert_eq!(response["selected"]["target"], "economy");
+ assert_eq!(response["fallbacks"], json!([]));
+ assert_eq!(response["response"]["model"], "model/weak");
+ assert_eq!(
+ response["response"]["choices"][0]["message"]["content"],
+ "ok"
+ );
+ assert_eq!(model_upstream.models().await, ["model/weak"]);
+ assert_eq!(judge_upstream.models().await, ["model/classifier"]);
+ Ok(())
+}
+
/// A critical tool error must reach the stage router's signal scorer, which reads
/// the decoded conversation. The endpoint records no inbound wire format, so a
/// scorer that parsed the raw body instead would find nothing and route every turn