Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(runner): let a route declare image input, so Codex stops omitting images (#567)

Oleksandr Pronin16c52fe7dd99
diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs
index 4b211ae81..c1fea1464 100644
--- a/crates/switchyard-runner/src/config.rs
+++ b/crates/switchyard-runner/src/config.rs
@@ -67,6 +67,7 @@ struct RouteConfig {
context_window: Option<u32>,
tool_calling: Option<bool>,
reasoning: Option<bool>,
+ vision: Option<bool>,
algorithm: AlgorithmSpec,
}
@@ -80,6 +81,7 @@ impl<'de> Deserialize<'de> for RouteConfig {
let context_window = take_optional(&mut table, "context_window")?;
let tool_calling = take_optional(&mut table, "tool_calling")?;
let reasoning = take_optional(&mut table, "reasoning")?;
+ let vision = take_optional(&mut table, "vision")?;
let algorithm = AlgorithmSpec::deserialize(toml::Value::Table(table))
.map_err(serde::de::Error::custom)?;
Ok(Self {
@@ -87,6 +89,7 @@ impl<'de> Deserialize<'de> for RouteConfig {
context_window,
tool_calling,
reasoning,
+ vision,
algorithm,
})
}
@@ -118,6 +121,7 @@ impl RouteConfig {
context_window: self.context_window,
tool_calling: self.tool_calling,
reasoning: self.reasoning,
+ vision: self.vision,
}
}
diff --git a/crates/switchyard-runner/src/route.rs b/crates/switchyard-runner/src/route.rs
index 857d51dfd..89f6e722c 100644
--- a/crates/switchyard-runner/src/route.rs
+++ b/crates/switchyard-runner/src/route.rs
@@ -26,6 +26,16 @@ pub struct ModelCapabilities {
/// probe this, so a route opts in via config; undeclared routes advertise as
/// non-reasoning to Codex (fail closed).
pub reasoning: Option<bool>,
+ /// Whether the routed model accepts image input. Declared per route for the same
+ /// reason as `reasoning`, and failing closed matters more here: a route may
+ /// resolve to a target with no vision at all.
+ ///
+ /// This is not cosmetic metadata. Codex reads `input_modalities` from the model
+ /// card and, when it reads text-only, replaces an attached image with the literal
+ /// text `image content omitted because you do not support image input` *before
+ /// sending*. An undeclared vision-capable route therefore loses the image in the
+ /// client, and the proxy never receives one to forward.
+ pub vision: Option<bool>,
}
/// Caller credential family required by a forwarded-auth route.
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 844931831..b92bd9aec 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -1478,6 +1478,7 @@ fn model_entry_json(model: &str, capabilities: ModelCapabilities) -> Value {
"capabilities": {
"streaming": true,
"tool_calling": capabilities.tool_calling,
+ "vision": capabilities.vision,
"context_window": capabilities.context_window,
"supported_inbound_formats": [
"openai-chat-completions",
@@ -1543,7 +1544,13 @@ fn codex_model_entry_json(model: &str, capabilities: ModelCapabilities, priority
"max_context_window": capabilities.context_window,
"effective_context_window_percent": 95,
"experimental_supported_tools": [],
- "input_modalities": ["text"],
+ // Codex omits an attached image client-side when this says text-only, so a
+ // route whose target can see must declare `vision = true`. Fails closed.
+ "input_modalities": if capabilities.vision.unwrap_or(false) {
+ json!(["text", "image"])
+ } else {
+ json!(["text"])
+ },
"supports_search_tool": false,
})
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index a45fa7151..b60921817 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -3474,3 +3474,75 @@ async fn responses_round_trips_codex_tool_namespaces() -> TestResult {
assert_eq!(completed["response"]["output"][0]["namespace"], "mcp__b");
Ok(())
}
+
+// Verifies a route declaring `vision = true` advertises image input, and that an
+// undeclared route still fails closed to text-only.
+//
+// This is not cosmetic metadata. Codex reads `input_modalities` from the model card
+// and, when it reads text-only, replaces an attached image with the literal text
+// "image content omitted because you do not support image input" before sending — so
+// a route whose target can see but which does not say so loses the image in the
+// client, and Switchyard never receives one to forward.
+#[tokio::test]
+async fn models_endpoint_advertises_image_input_only_for_vision_routes() -> TestResult {
+ const CONFIG: &str = r#"
+schema_version = 1
+
+[llm_clients.shared]
+format = "openai_responses"
+base_url = "http://127.0.0.1:1/v1"
+
+[targets.shared]
+id = "shared-model"
+llm_client = "shared"
+
+[routes.sees]
+id = "sees"
+type = "passthrough"
+target = "shared"
+vision = true
+
+[routes.blind]
+id = "blind"
+type = "passthrough"
+target = "shared"
+"#;
+ let app = build_switchyard_router(load_test_config(CONFIG)?);
+ let models = send(&app, "GET", "/v1/models", None).await?;
+ assert_eq!(models.status, StatusCode::OK);
+ let body = models.json()?;
+
+ let codex_metadata = body["models"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default()
+ .iter()
+ .filter_map(|entry| {
+ entry["slug"]
+ .as_str()
+ .map(|slug| (slug.to_string(), entry.clone()))
+ })
+ .collect::<BTreeMap<_, _>>();
+ assert_eq!(
+ codex_metadata["sees"]["input_modalities"],
+ json!(["text", "image"])
+ );
+ assert_eq!(codex_metadata["blind"]["input_modalities"], json!(["text"]));
+
+ // The OpenAI `data` entry reports the raw Option, so an undeclared route stays
+ // distinguishable from one that declared `false`.
+ let capabilities = body["data"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default()
+ .iter()
+ .filter_map(|entry| {
+ entry["id"]
+ .as_str()
+ .map(|id| (id.to_string(), entry["capabilities"].clone()))
+ })
+ .collect::<BTreeMap<_, _>>();
+ assert_eq!(capabilities["sees"]["vision"], json!(true));
+ assert_eq!(capabilities["blind"]["vision"], json!(null));
+ Ok(())
+}
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 92d2b2b75..0193ca680 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -101,6 +101,7 @@ Every route takes the common keys below, plus the keys for its type.
| `context_window` | No | unset | Positive token count advertised for this route by `GET /v1/models`. Unset values appear as `null`. This does not enforce a request limit. |
| `tool_calling` | No | unset | Whether `GET /v1/models` advertises tool-calling support for this route. Unset values appear as `null`. |
| `reasoning` | No | unset | Whether `GET /v1/models` advertises reasoning support to Codex direct-provider discovery. Unset routes are advertised as non-reasoning. |
+| `vision` | No | unset | Whether `GET /v1/models` advertises **image input** to Codex direct-provider discovery. Unset routes are advertised as text-only. This is not cosmetic: Codex reads `input_modalities` from the model card and, when it reads text-only, replaces an attached image with the text `image content omitted because you do not support image input` **before sending**, so a route whose target can see but which does not declare `vision = true` loses the image in the client. Declare it only when every target the route can select accepts images. |
### `noop`