Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): report declared model capabilities on /v1/models (#200)

Elyas Mehtabuddin17b0e174f28b
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index d43060cf0..1a0627147 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -21,7 +21,7 @@ use switchyard_llm_client::{
};
use switchyard_protocol::RoutedLlmClient;
-use crate::{ServerError, ServerResult, ServerState};
+use crate::{ModelCapabilities, ServerError, ServerResult, ServerState};
const SUPPORTED_SCHEMA_VERSION: u32 = 1;
const MAX_CONFIGURED_RETRIES: u32 = 10;
@@ -91,12 +91,16 @@ impl ServerConfig {
for (route_name, config) in &self.routes {
validate_value("route name", route_name)?;
validate_value(&format!("route {route_name} id"), config.id())?;
- routes.push((
- config.id().to_string(),
- build_algorithm(route_name, config, &targets)?,
- ));
+ let capabilities = config.capabilities();
+ if capabilities.context_window == Some(0) {
+ return Err(ServerError::new(format!(
+ "route {route_name} context_window must be greater than zero"
+ )));
+ }
+ let algorithm = build_algorithm(route_name, config, &targets)?;
+ routes.push((config.id().to_string(), algorithm, capabilities));
}
- ServerState::new(routes)
+ ServerState::new_with_capabilities(routes)
}
fn build_clients(&self) -> ServerResult<BTreeMap<String, Arc<dyn RoutedLlmClient>>> {
@@ -260,19 +264,35 @@ struct CustomClassifierRouteConfig {
enum RouteConfig {
Noop {
id: String,
+ #[serde(default)]
+ context_window: Option<u32>,
+ #[serde(default)]
+ tool_calling: Option<bool>,
},
Random {
id: String,
+ #[serde(default)]
+ context_window: Option<u32>,
+ #[serde(default)]
+ tool_calling: Option<bool>,
targets: Vec<String>,
weights: Option<Vec<f64>>,
seed: Option<u64>,
},
Passthrough {
id: String,
+ #[serde(default)]
+ context_window: Option<u32>,
+ #[serde(default)]
+ tool_calling: Option<bool>,
target: String,
},
LlmClassifier {
id: String,
+ #[serde(default)]
+ context_window: Option<u32>,
+ #[serde(default)]
+ tool_calling: Option<bool>,
classifier_target: String,
#[serde(default)]
mode: Option<ClassifierMode>,
@@ -309,6 +329,10 @@ enum RouteConfig {
},
StageRouter {
id: String,
+ #[serde(default)]
+ context_window: Option<u32>,
+ #[serde(default)]
+ tool_calling: Option<bool>,
capable_target: String,
efficient_target: String,
/// Tier a turn falls back to when the signals are not confident.
@@ -373,7 +397,7 @@ impl RouteConfig {
fn id(&self) -> &str {
use RouteConfig::*;
match self {
- Noop { id }
+ Noop { id, .. }
| Random { id, .. }
| LlmClassifier { id, .. }
| Passthrough { id, .. }
@@ -381,6 +405,40 @@ impl RouteConfig {
}
}
+ fn capabilities(&self) -> ModelCapabilities {
+ use RouteConfig::*;
+ match self {
+ Noop {
+ context_window,
+ tool_calling,
+ ..
+ }
+ | Random {
+ context_window,
+ tool_calling,
+ ..
+ }
+ | Passthrough {
+ context_window,
+ tool_calling,
+ ..
+ }
+ | LlmClassifier {
+ context_window,
+ tool_calling,
+ ..
+ }
+ | StageRouter {
+ context_window,
+ tool_calling,
+ ..
+ } => ModelCapabilities {
+ context_window: *context_window,
+ tool_calling: *tool_calling,
+ },
+ }
+ }
+
fn classifier_mode(&self, route_name: &str) -> ServerResult<LlmClassifierModeConfig> {
let Self::LlmClassifier {
mode,
@@ -997,6 +1055,12 @@ target = "weak"
);
assert!(error_message(&unknown_classifier_field).contains("unknown field"));
+ let target_capability = VALID_CONFIG.replace(
+ "llm_client = \"responses\"",
+ "llm_client = \"responses\"\ncontext_window = 1000000",
+ );
+ assert!(error_message(&target_capability).contains("unknown field `context_window`"));
+
let unknown_algorithm = VALID_CONFIG.replace("type = \"noop\"", "type = \"imaginary\"");
assert!(error_message(&unknown_algorithm).contains("unknown variant"));
}
@@ -1090,6 +1154,13 @@ classifier_magic = true
VALID_CONFIG.replace("[targets.strong]", "[targets.\" strong \"]"),
"target name must be non-empty and have no surrounding whitespace",
),
+ (
+ VALID_CONFIG.replace(
+ "targets = [\"strong\", \"weak\"]",
+ "targets = [\"strong\", \"weak\"]\ncontext_window = 0",
+ ),
+ "route random context_window must be greater than zero",
+ ),
];
for (toml, expected) in cases {
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 97c253b4d..9196f0844 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -84,10 +84,27 @@ impl Error for ServerError {}
/// Result returned by server setup and lifecycle operations.
pub type ServerResult<T> = std::result::Result<T, ServerError>;
+/// Capabilities that one route advertises on `GET /v1/models`.
+///
+/// An unset capability is undeclared and serializes as `null`.
+#[derive(Clone, Copy, Default)]
+struct ModelCapabilities {
+ context_window: Option<u32>,
+ tool_calling: Option<bool>,
+}
+
+/// A registered route: the libsy algorithm that serves it and the capabilities
+/// advertised for it on `GET /v1/models`. One entry owns both so the routing
+/// runtime and the model listing can never drift apart.
+struct RouteEntry {
+ algorithm: Arc<dyn Algorithm>,
+ capabilities: ModelCapabilities,
+}
+
/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
- routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,
+ routes: Arc<BTreeMap<String, RouteEntry>>,
metrics: prometheus::Registry,
stats: StatsAccumulator,
routing_log: Option<SharedRoutingLog>,
@@ -132,14 +149,28 @@ impl ServerState {
/// Creates server state from route model IDs and their libsy algorithms.
pub fn new(
routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,
+ ) -> ServerResult<Self> {
+ Self::new_with_capabilities(
+ routes
+ .into_iter()
+ .map(|(model, algorithm)| (model, algorithm, ModelCapabilities::default())),
+ )
+ }
+
+ fn new_with_capabilities(
+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>, ModelCapabilities)>,
) -> ServerResult<Self> {
let mut entries = BTreeMap::new();
- for (model, algorithm) in routes {
+ for (model, algorithm, capabilities) in routes {
let model = model.trim();
if model.is_empty() {
return Err(ServerError::new("route model must not be empty"));
}
- if entries.insert(model.to_string(), algorithm).is_some() {
+ let entry = RouteEntry {
+ algorithm,
+ capabilities,
+ };
+ if entries.insert(model.to_string(), entry).is_some() {
return Err(ServerError::new(format!("duplicate route model {model}")));
}
}
@@ -168,7 +199,9 @@ impl ServerState {
}
fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {
- self.routes.get(model).map(Arc::clone)
+ self.routes
+ .get(model)
+ .map(|entry| Arc::clone(&entry.algorithm))
}
}
@@ -827,7 +860,12 @@ fn error_response(
}
async fn models(State(state): State<ServerState>) -> Json<Value> {
- Json(model_list_payload(state.models()))
+ Json(model_list_payload(
+ state
+ .routes
+ .iter()
+ .map(|(model, entry)| (model.as_str(), entry.capabilities)),
+ ))
}
async fn get_stats(State(state): State<ServerState>) -> Json<StatsSnapshot> {
@@ -907,13 +945,16 @@ async fn not_found() -> Response {
)
}
-fn model_list_payload<'a>(models: impl IntoIterator<Item = &'a str>) -> Value {
- let model_ids = models.into_iter().map(str::to_string).collect::<Vec<_>>();
- let first_id = model_ids.first().cloned();
- let last_id = model_ids.last().cloned();
+fn model_list_payload<'a>(
+ entries: impl IntoIterator<Item = (&'a str, ModelCapabilities)>,
+) -> Value {
+ let entries = entries.into_iter().collect::<Vec<_>>();
+ let model_ids = entries.iter().map(|(model, _)| *model).collect::<Vec<_>>();
+ let first_id = model_ids.first().copied();
+ let last_id = model_ids.last().copied();
json!({
"object": "list",
- "data": model_ids.iter().map(|model| model_entry_json(model)).collect::<Vec<_>>(),
+ "data": entries.iter().map(|(model, caps)| model_entry_json(model, *caps)).collect::<Vec<_>>(),
"first_id": first_id,
"last_id": last_id,
"has_more": false,
@@ -922,7 +963,7 @@ fn model_list_payload<'a>(models: impl IntoIterator<Item = &'a str>) -> Value {
})
}
-fn model_entry_json(model: &str) -> Value {
+fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
json!({
"id": model,
"object": "model",
@@ -932,8 +973,8 @@ fn model_entry_json(model: &str) -> Value {
"display_name": model,
"capabilities": {
"streaming": true,
- "tool_calling": null,
- "context_window": null,
+ "tool_calling": capabilities.tool_calling,
+ "context_window": capabilities.context_window,
"supported_inbound_formats": [
"openai-chat-completions",
"openai-responses",
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 406110db9..3860cdc85 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1101,6 +1101,58 @@ async fn routes_dispatch_and_discovery_endpoints_are_stable() -> TestResult {
Ok(())
}
+#[tokio::test]
+async fn models_endpoint_reports_declared_route_capabilities_and_null_when_undeclared() -> TestResult
+{
+ const CONFIG: &str = r#"
+schema_version = 1
+
+[llm_clients.primary]
+format = "openai_chat"
+base_url = "https://example.test/v1"
+
+[targets.shared]
+id = "nvidia/deepseek-ai/deepseek-v4-pro"
+llm_client = "primary"
+
+[routes.declared]
+id = "declared"
+type = "passthrough"
+target = "shared"
+context_window = 1000000
+tool_calling = true
+
+[routes.restricted]
+id = "restricted"
+type = "passthrough"
+target = "shared"
+context_window = 262000
+tool_calling = false
+
+[routes.undeclared]
+id = "undeclared"
+type = "passthrough"
+target = "shared"
+"#;
+ let app = build_switchyard_router(load_test_config(CONFIG)?);
+ let models = send(&app, "GET", "/v1/models", None).await?;
+ assert_eq!(models.status, StatusCode::OK);
+ let body = models.json()?;
+ let data = body["data"].as_array().cloned().unwrap_or_default();
+ let capabilities = data
+ .iter()
+ .filter_map(|entry| entry["id"].as_str().map(|id| (id, &entry["capabilities"])))
+ .collect::<BTreeMap<_, _>>();
+
+ assert_eq!(capabilities["declared"]["context_window"], json!(1_000_000));
+ assert_eq!(capabilities["declared"]["tool_calling"], json!(true));
+ assert_eq!(capabilities["restricted"]["context_window"], json!(262_000));
+ assert_eq!(capabilities["restricted"]["tool_calling"], json!(false));
+ assert_eq!(capabilities["undeclared"]["context_window"], json!(null));
+ assert_eq!(capabilities["undeclared"]["tool_calling"], json!(null));
+ Ok(())
+}
+
#[tokio::test]
async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestResult {
let (upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 75e4eac9d..45092727e 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -56,7 +56,14 @@ must exist and be non-empty when the server loads.
## `[routes.<name>]`
-Every route takes `id` and `type`, plus the keys for that type.
+Every route takes the common keys below, plus the keys for its type.
+
+| Key | Required | Default | Meaning |
+|---|:---:|---|---|
+| `id` | Yes | — | Public model ID that callers send in requests. |
+| `type` | Yes | — | Routing algorithm for this route. |
+| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. |
+| `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. |
### `noop`