Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(server): report declared model capabilities on /v1/models (#200)
Elyas Mehtabuddin17b0e174f28b
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex d43060cf0..1a0627147 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -21,7 +21,7 @@ use switchyard_llm_client::{};use switchyard_protocol::RoutedLlmClient;-use crate::{ServerError, ServerResult, ServerState};+use crate::{ModelCapabilities, ServerError, ServerResult, ServerState};const SUPPORTED_SCHEMA_VERSION: u32 = 1;const MAX_CONFIGURED_RETRIES: u32 = 10;@@ -91,12 +91,16 @@ impl ServerConfig {for (route_name, config) in &self.routes {validate_value("route name", route_name)?;validate_value(&format!("route {route_name} id"), config.id())?;- routes.push((- config.id().to_string(),- build_algorithm(route_name, config, &targets)?,- ));+ let capabilities = config.capabilities();+ if capabilities.context_window == Some(0) {+ return Err(ServerError::new(format!(+ "route {route_name} context_window must be greater than zero"+ )));+ }+ let algorithm = build_algorithm(route_name, config, &targets)?;+ routes.push((config.id().to_string(), algorithm, capabilities));}- ServerState::new(routes)+ ServerState::new_with_capabilities(routes)}fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<dyn RoutedLlmClient>>> {@@ -260,19 +264,35 @@ struct CustomClassifierRouteConfig {enum RouteConfig {Noop {id: String,+ #[serde(default)]+ context_window: Option<u32>,+ #[serde(default)]+ tool_calling: Option<bool>,},Random {id: String,+ #[serde(default)]+ context_window: Option<u32>,+ #[serde(default)]+ tool_calling: Option<bool>,targets: Vec<String>,weights: Option<Vec<f64>>,seed: Option<u64>,},Passthrough {id: String,+ #[serde(default)]+ context_window: Option<u32>,+ #[serde(default)]+ tool_calling: Option<bool>,target: String,},LlmClassifier {id: String,+ #[serde(default)]+ context_window: Option<u32>,+ #[serde(default)]+ tool_calling: Option<bool>,classifier_target: String,#[serde(default)]mode: Option<ClassifierMode>,@@ -309,6 +329,10 @@ enum RouteConfig {},StageRouter {id: String,+ #[serde(default)]+ context_window: Option<u32>,+ #[serde(default)]+ tool_calling: Option<bool>,capable_target: String,efficient_target: String,/// Tier a turn falls back to when the signals are not confident.@@ -373,7 +397,7 @@ impl RouteConfig {fn id(&self) -> &str {use RouteConfig::*;match self {- Noop { id }+ Noop { id, .. }| Random { id, .. }| LlmClassifier { id, .. }| Passthrough { id, .. }@@ -381,6 +405,40 @@ impl RouteConfig {}}+ fn capabilities(&self) -> ModelCapabilities {+ use RouteConfig::*;+ match self {+ Noop {+ context_window,+ tool_calling,+ ..+ }+ | Random {+ context_window,+ tool_calling,+ ..+ }+ | Passthrough {+ context_window,+ tool_calling,+ ..+ }+ | LlmClassifier {+ context_window,+ tool_calling,+ ..+ }+ | StageRouter {+ context_window,+ tool_calling,+ ..+ } => ModelCapabilities {+ context_window: *context_window,+ tool_calling: *tool_calling,+ },+ }+ }+fn classifier_mode(&self, route_name: &str) -> ServerResult<LlmClassifierModeConfig> {let Self::LlmClassifier {mode,@@ -997,6 +1055,12 @@ target = "weak");assert!(error_message(&unknown_classifier_field).contains("unknown field"));+ let target_capability = VALID_CONFIG.replace(+ "llm_client = \"responses\"",+ "llm_client = \"responses\"\ncontext_window = 1000000",+ );+ assert!(error_message(&target_capability).contains("unknown field `context_window`"));+let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");assert!(error_message(&unknown_algorithm).contains("unknown variant"));}@@ -1090,6 +1154,13 @@ classifier_magic = trueVALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),"target name must be non-empty and have no surrounding whitespace",),+ (+ VALID_CONFIG.replace(+ "targets = [\"strong\", \"weak\"]",+ "targets = [\"strong\", \"weak\"]\ncontext_window = 0",+ ),+ "route random context_window must be greater than zero",+ ),];for (toml, expected) in cases {diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 97c253b4d..9196f0844 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -84,10 +84,27 @@ impl Error for ServerError {}/// Result returned by server setup and lifecycle operations.pub type ServerResult<T> = std::result::Result<T, ServerError>;+/// Capabilities that one route advertises on `GET /v1/models`.+///+/// An unset capability is undeclared and serializes as `null`.+#[derive(Clone, Copy, Default)]+struct ModelCapabilities {+ context_window: Option<u32>,+ tool_calling: Option<bool>,+}++/// A registered route: the libsy algorithm that serves it and the capabilities+/// advertised for it on `GET /v1/models`. One entry owns both so the routing+/// runtime and the model listing can never drift apart.+struct RouteEntry {+ algorithm: Arc<dyn Algorithm>,+ capabilities: ModelCapabilities,+}+/// Shared server state used by all endpoint handlers.#[derive(Clone)]pub struct ServerState {- routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,+ routes: Arc<BTreeMap<String, RouteEntry>>,metrics: prometheus::Registry,stats: StatsAccumulator,routing_log: Option<SharedRoutingLog>,@@ -132,14 +149,28 @@ impl ServerState {/// Creates server state from route model IDs and their libsy algorithms.pub fn new(routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,+ ) -> ServerResult<Self> {+ Self::new_with_capabilities(+ routes+ .into_iter()+ .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default())),+ )+ }++ fn new_with_capabilities(+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>, ModelCapabilities)>,) -> ServerResult<Self> {let mut entries = BTreeMap::new();- for (model, algorithm) in routes {+ for (model, algorithm, capabilities) in routes {let model = model.trim();if model.is_empty() {return Err(ServerError::new("route model must not be empty"));}- if entries.insert(model.to_string(), algorithm).is_some() {+ let entry = RouteEntry {+ algorithm,+ capabilities,+ };+ if entries.insert(model.to_string(), entry).is_some() {return Err(ServerError::new(format!("duplicate route model {model}")));}}@@ -168,7 +199,9 @@ impl ServerState {}fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {- self.routes.get(model).map(Arc::clone)+ self.routes+ .get(model)+ .map(|entry| Arc::clone(&entry.algorithm))}}@@ -827,7 +860,12 @@ fn error_response(}async fn models(State(state): State<ServerState>) -> Json<Value> {- Json(model_list_payload(state.models()))+ Json(model_list_payload(+ state+ .routes+ .iter()+ .map(|(model, entry)| (model.as_str(), entry.capabilities)),+ ))}async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {@@ -907,13 +945,16 @@ async fn not_found() -> Response {)}-fn model_list_payload<'a>(models: impl IntoIterator<Item = &'a str>) -> Value {- let model_ids = models.into_iter().map(str::to_string).collect::<Vec<_>>();- let first_id = model_ids.first().cloned();- let last_id = model_ids.last().cloned();+fn model_list_payload<'a>(+ entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,+) -> Value {+ let entries = entries.into_iter().collect::<Vec<_>>();+ let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();+ let first_id = model_ids.first().copied();+ let last_id = model_ids.last().copied();json!({"object": "list",- "data": model_ids.iter().map(|model| model_entry_json(model)).collect::<Vec<_>>(),+ "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),"first_id": first_id,"last_id": last_id,"has_more": false,@@ -922,7 +963,7 @@ fn model_list_payload<'a>(models: impl IntoIterator<Item = &'a str>) -> Value {})}-fn model_entry_json(model: &str) -> Value {+fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {json!({"id": model,"object": "model",@@ -932,8 +973,8 @@ fn model_entry_json(model: &str) -> Value {"display_name": model,"capabilities": {"streaming": true,- "tool_calling": null,- "context_window": null,+ "tool_calling": capabilities.tool_calling,+ "context_window": capabilities.context_window,"supported_inbound_formats": ["openai-chat-completions","openai-responses",diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 406110db9..3860cdc85 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -1101,6 +1101,58 @@ async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {Ok(())}+#[tokio::test]+async fn models_endpoint_reports_declared_route_capabilities_and_null_when_undeclared() -> TestResult+{+ const CONFIG: &str = r#"+schema_version = 1++[llm_clients.primary]+format = "openai_chat"+base_url = "https://example.test/v1"++[targets.shared]+id = "nvidia/deepseek-ai/deepseek-v4-pro"+llm_client = "primary"++[routes.declared]+id = "declared"+type = "passthrough"+target = "shared"+context_window = 1000000+tool_calling = true++[routes.restricted]+id = "restricted"+type = "passthrough"+target = "shared"+context_window = 262000+tool_calling = false++[routes.undeclared]+id = "undeclared"+type = "passthrough"+target = "shared"+"#;+ let app = build_switchyard_router(load_test_config(CONFIG)?);+ let models = send(&app, "GET", "/v1/models", None).await?;+ assert_eq!(models.status, StatusCode::OK);+ let body = models.json()?;+ let data = body["data"].as_array().cloned().unwrap_or_default();+ let capabilities = data+ .iter()+ .filter_map(|entry| entry["id"].as_str().map(|id| (id, &entry["capabilities"])))+ .collect::<BTreeMap<_, _>>();++ assert_eq!(capabilities["declared"]["context_window"], json!(1_000_000));+ assert_eq!(capabilities["declared"]["tool_calling"], json!(true));+ assert_eq!(capabilities["restricted"]["context_window"], json!(262_000));+ assert_eq!(capabilities["restricted"]["tool_calling"], json!(false));+ assert_eq!(capabilities["undeclared"]["context_window"], json!(null));+ assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null));+ Ok(())+}+#[tokio::test]async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult {let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 75e4eac9d..45092727e 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -56,7 +56,14 @@ must exist and be non-empty when the server loads.## `[routes.<name>]`-Every route takes `id` and `type`, plus the keys for that type.+Every route takes the common keys below, plus the keys for its type.++| Key | Required | Default | Meaning |+|---|:---:|---|---|+| `id` | Yes | — | Public model ID that callers send in requests. |+| `type` | Yes | — | Routing algorithm for this route. |+| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. |+| `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. |### `noop`