Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(runner): let a route declare image input, so Codex stops omitting images (#567)
Oleksandr Pronin16c52fe7dd99
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rsindex 4b211ae81..c1fea1464 100644--- a/crates/switchyard-runner/src/config.rs+++ b/crates/switchyard-runner/src/config.rs@@ -67,6 +67,7 @@ struct RouteConfig {context_window: Option<u32>,tool_calling: Option<bool>,reasoning: Option<bool>,+ vision: Option<bool>,algorithm: AlgorithmSpec,}@@ -80,6 +81,7 @@ impl<'de> Deserialize<'de> for RouteConfig {let context_window = take_optional(&mut table, "context_window")?;let tool_calling = take_optional(&mut table, "tool_calling")?;let reasoning = take_optional(&mut table, "reasoning")?;+ let vision = take_optional(&mut table, "vision")?;let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table)).map_err(serde::de::Error::custom)?;Ok(Self {@@ -87,6 +89,7 @@ impl<'de> Deserialize<'de> for RouteConfig {context_window,tool_calling,reasoning,+ vision,algorithm,})}@@ -118,6 +121,7 @@ impl RouteConfig {context_window: self.context_window,tool_calling: self.tool_calling,reasoning: self.reasoning,+ vision: self.vision,}}diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rsindex 857d51dfd..89f6e722c 100644--- a/crates/switchyard-runner/src/route.rs+++ b/crates/switchyard-runner/src/route.rs@@ -26,6 +26,16 @@ pub struct ModelCapabilities {/// probe this, so a route opts in via config; undeclared routes advertise as/// non-reasoning to Codex (fail closed).pub reasoning: Option<bool>,+ /// Whether the routed model accepts image input. Declared per route for the same+ /// reason as `reasoning`, and failing closed matters more here: a route may+ /// resolve to a target with no vision at all.+ ///+ /// This is not cosmetic metadata. Codex reads `input_modalities` from the model+ /// card and, when it reads text-only, replaces an attached image with the literal+ /// text `image content omitted because you do not support image input` *before+ /// sending*. An undeclared vision-capable route therefore loses the image in the+ /// client, and the proxy never receives one to forward.+ pub vision: Option<bool>,}/// Caller credential family required by a forwarded-auth route.diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 844931831..b92bd9aec 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -1478,6 +1478,7 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {"capabilities": {"streaming": true,"tool_calling": capabilities.tool_calling,+ "vision": capabilities.vision,"context_window": capabilities.context_window,"supported_inbound_formats": ["openai-chat-completions",@@ -1543,7 +1544,13 @@ fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority"max_context_window": capabilities.context_window,"effective_context_window_percent": 95,"experimental_supported_tools": [],- "input_modalities": ["text"],+ // Codex omits an attached image client-side when this says text-only, so a+ // route whose target can see must declare `vision = true`. Fails closed.+ "input_modalities": if capabilities.vision.unwrap_or(false) {+ json!(["text", "image"])+ } else {+ json!(["text"])+ },"supports_search_tool": false,})}diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex a45fa7151..b60921817 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -3474,3 +3474,75 @@ async fn responses_round_trips_codex_tool_namespaces() -> TestResult {assert_eq!(completed["response"]["output"][0]["namespace"], "mcp__b");Ok(())}++// Verifies a route declaring `vision = true` advertises image input, and that an+// undeclared route still fails closed to text-only.+//+// This is not cosmetic metadata. Codex reads `input_modalities` from the model card+// and, when it reads text-only, replaces an attached image with the literal text+// "image content omitted because you do not support image input" before sending — so+// a route whose target can see but which does not say so loses the image in the+// client, and Switchyard never receives one to forward.+#[tokio::test]+async fn models_endpoint_advertises_image_input_only_for_vision_routes() -> TestResult {+ const CONFIG: &str = r#"+schema_version = 1++[llm_clients.shared]+format = "openai_responses"+base_url = "http://127.0.0.1:1/v1"++[targets.shared]+id = "shared-model"+llm_client = "shared"++[routes.sees]+id = "sees"+type = "passthrough"+target = "shared"+vision = true++[routes.blind]+id = "blind"+type = "passthrough"+target = "shared"+"#;+ let app = build_switchyard_router(load_test_config(CONFIG)?);+ let models = send(&app, "GET", "/v1/models", None).await?;+ assert_eq!(models.status, StatusCode::OK);+ let body = models.json()?;++ let codex_metadata = body["models"]+ .as_array()+ .cloned()+ .unwrap_or_default()+ .iter()+ .filter_map(|entry| {+ entry["slug"]+ .as_str()+ .map(|slug| (slug.to_string(), entry.clone()))+ })+ .collect::<BTreeMap<_, _>>();+ assert_eq!(+ codex_metadata["sees"]["input_modalities"],+ json!(["text", "image"])+ );+ assert_eq!(codex_metadata["blind"]["input_modalities"], json!(["text"]));++ // The OpenAI `data` entry reports the raw Option, so an undeclared route stays+ // distinguishable from one that declared `false`.+ let capabilities = body["data"]+ .as_array()+ .cloned()+ .unwrap_or_default()+ .iter()+ .filter_map(|entry| {+ entry["id"]+ .as_str()+ .map(|id| (id.to_string(), entry["capabilities"].clone()))+ })+ .collect::<BTreeMap<_, _>>();+ assert_eq!(capabilities["sees"]["vision"], json!(true));+ assert_eq!(capabilities["blind"]["vision"], json!(null));+ Ok(())+}diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 92d2b2b75..0193ca680 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -101,6 +101,7 @@ Every route takes the common keys below, plus the keys for its type.| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. || `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. || `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. |+| `vision` | No | unset | Whether `GET /v1/models` advertises **image input** to Codex direct-provider discovery. Unset routes are advertised as text-only. This is not cosmetic: Codex reads `input_modalities` from the model card and, when it reads text-only, replaces an attached image with the text `image content omitted because you do not support image input` **before sending**, so a route whose target can see but which does not declare `vision = true` loses the image in the client. Declare it only when every target the route can select accepts images. |### `noop`