Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): report the served model on every response path (#169)

eric-liu-nvidiaa3a3d319ede8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 56661edfd..63195a865 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -23,6 +23,20 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
no `type: model` equivalent, so list the model ids you want as explicit
`type: model` routes.
+### Fixed
+
+- **Response `model` now names the model that actually served the request**, on
+ every serving path and wire format. Streamed Anthropic and Responses replies,
+ and every libsy-served reply, previously echoed the model id the client
+ requested — for a route bundle whose key is an alias, that meant the alias
+ rather than the routed target, so trajectories, dashboards, and client UIs
+ labelled routed turns with the route name. The routed model was already
+ reported by `x-model-router-selected-model`, `x-switchyard-selected-model`,
+ `/v1/routing/stats`, and Intake's `served_model`; the response body now agrees
+ with them. Streamed OpenAI Chat replies report the routed target instead of
+ the provider's own id, and no longer fall back to `"unknown"` when a provider
+ omits `model` on delta chunks.
+
## [0.1.0] — Initial release
First public release of Switchyard — a typed, composable control plane for LLM
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 565d3464a..4b3a36ec5 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -411,7 +411,8 @@ impl TranslatingLlmClient {
/// encodes the neutral response back into `wire_format`. The result is a buffered
/// [`RawResponse::Buffered`] JSON body or a streamed [`RawResponse::Stream`] of
/// wire events (the caller frames the stream as SSE). The response's `model` is
- /// restamped with the model the request asked for, never the upstream id.
+ /// restamped with the model that actually served the call, so the body names the
+ /// model that answered rather than the route the caller addressed.
///
/// `http_headers` are carried through as the request's
/// [`Metadata::http_headers`] and forwarded to the upstream (minus the reserved
@@ -426,9 +427,12 @@ impl TranslatingLlmClient {
) -> Result<RawResponse> {
let llm_request = decode_request(wire_format, &raw_http_request)
.map_err(|error| LlmClientError::RequestTranslation(error.to_string()))?;
- // The model the client asked for; restamped onto the response so it never
- // leaks the upstream id.
- let requested_model = llm_request.model.clone();
+ // The model that serves the call — the rewrite target when the caller pinned
+ // one, else the request's own model. Mirrors `call_rewrite_model`'s own
+ // resolution so the response names whoever answered.
+ let served_model = model
+ .map(str::to_string)
+ .or_else(|| llm_request.model.clone());
let request = Request {
llm_request,
@@ -449,12 +453,12 @@ impl TranslatingLlmClient {
match response.llm_response {
LlmResponse::Agg(agg) => {
let body =
- encode_aggregated_response(&agg, wire_format, requested_model.as_deref())
+ encode_aggregated_response(&agg, wire_format, served_model.as_deref())
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
Ok(RawResponse::Buffered(body))
}
LlmResponse::Stream(chunks) => {
- let events = encode_stream(chunks, wire_format, requested_model)?;
+ let events = encode_stream(chunks, wire_format, served_model)?;
Ok(RawResponse::Stream(events))
}
}
@@ -1618,7 +1622,7 @@ mod tests {
}
// Raw path, buffered: decode an OpenAI Chat body -> call -> encode back to OpenAI
- // Chat JSON, with the client-facing `model` restamped over the upstream id.
+ // Chat JSON, with the served `model` restamped over the id the caller addressed.
#[tokio::test]
async fn call_rewrite_model_raw_round_trips_buffered_json(
) -> std::result::Result<(), Box<dyn Error + Sync + Send + 'static>> {
@@ -1657,8 +1661,8 @@ mod tests {
};
assert_eq!(body["choices"][0]["message"]["content"], "Hi there");
- // The client sees the model it asked for, not the upstream "gpt".
- assert_eq!(body["model"], "client-facing");
+ // The client sees the model that answered, not the "client-facing" route id.
+ assert_eq!(body["model"], "gpt");
Ok(())
}
@@ -1705,6 +1709,9 @@ mod tests {
.filter_map(|event| event["choices"][0]["delta"]["content"].as_str())
.collect();
assert_eq!(content, "Hello world");
+ // The mock frames carry no `model`, so every chunk's model comes from the
+ // served id rather than the "unknown" fallback or the caller's route id.
+ assert!(events.iter().all(|event| event["model"] == "gpt"));
Ok(())
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index aa2b4fd1d..edc6107bb 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -293,7 +293,7 @@ async fn anthropic_count_tokens(
Ok(body) => body,
Err(message) => return invalid_body_error(message),
};
- let (algorithm, request, _) = match resolve_route(
+ let (algorithm, request) = match resolve_route(
&state,
metadata_from_headers(&headers),
body,
@@ -373,9 +373,8 @@ fn llm_json_body(
/// Decode `body`, resolve the route named by its `model`, and build the
/// [`Request`]. Shared by the completion and `count_tokens` handlers. Returns
-/// the resolved algorithm, the built request, and the requested route model —
-/// or an error [`Response`] (invalid body, empty `model` → 400, unknown route →
-/// 404).
+/// the resolved algorithm and the built request — or an error [`Response`]
+/// (invalid body, empty `model` → 400, unknown route → 404).
// Both callers immediately return the `Err(Response)` as the HTTP response, so
// the large error type is intentional, not propagated up a call stack.
#[allow(clippy::type_complexity, clippy::result_large_err)]
@@ -384,7 +383,7 @@ fn resolve_route(
metadata: Metadata,
body: Value,
wire_format: WireFormat,
-) -> std::result::Result<(Arc<dyn Algorithm>, Request, String), Response> {
+) -> std::result::Result<(Arc<dyn Algorithm>, Request), Response> {
let llm_request = decode_request(wire_format, &body)
.map_err(|error| invalid_body_error(error.to_string()))?;
let requested_model = llm_request
@@ -412,7 +411,7 @@ fn resolve_route(
raw_request: Some(body),
metadata: Some(metadata),
};
- Ok((algorithm, request, requested_model))
+ Ok((algorithm, request))
}
async fn handle_llm_request(
@@ -422,16 +421,19 @@ async fn handle_llm_request(
body: Value,
wire_format: WireFormat,
) -> Response {
- let (algorithm, request, requested_model) =
- match resolve_route(&state, metadata, body, wire_format) {
- Ok(resolved) => resolved,
- Err(response) => return response,
- };
+ let (algorithm, request) = match resolve_route(&state, metadata, body, wire_format) {
+ Ok(resolved) => resolved,
+ Err(response) => return response,
+ };
let (trace, response) = match algorithm.run(Context::default(), request).await {
Ok(result) => result,
Err(error) => return algorithm_error(error),
};
- let response = if let Some(decision) = trace.last() {
+ // Metrics, response body, and routing header all read the same decision, so
+ // the model they name can never disagree. An empty trace leaves the body with
+ // the id the upstream reported.
+ let decision = trace.last();
+ let response = if let Some(decision) = decision {
usage_metrics::observe(
response,
decision.selected_model(),
@@ -442,11 +444,12 @@ async fn handle_llm_request(
response
};
- let mut response = match into_http_response(response, wire_format, Some(requested_model)) {
+ let served_model = decision.map(|decision| decision.selected_model().to_string());
+ let mut response = match into_http_response(response, wire_format, served_model) {
Ok(response) => response,
Err(error) => return server_error(error.to_string()),
};
- if let Some(decision) = trace.last() {
+ if let Some(decision) = decision {
attach_routing_headers(&mut response, decision.as_ref());
}
response
diff --git a/crates/switchyard-server/src/response.rs b/crates/switchyard-server/src/response.rs
index 9488d1764..b03deeea9 100644
--- a/crates/switchyard-server/src/response.rs
+++ b/crates/switchyard-server/src/response.rs
@@ -14,21 +14,23 @@ use crate::sse::frame_stream;
type BoxError = Box<dyn Error + Send + Sync>;
-/// Encodes a libsy response into the endpoint's wire format.
+/// Encodes a libsy response into the endpoint's wire format, reporting
+/// `served_model` as the response model so the body names the model that
+/// answered rather than the route the caller addressed.
pub(crate) fn into_http_response(
response: AlgorithmResponse,
target_format: WireFormat,
- requested_model: Option<String>,
+ served_model: Option<String>,
) -> Result<HttpResponse, BoxError> {
match response.llm_response {
LlmResponse::Agg(response) => Ok(Json(encode_aggregated_response(
&response,
target_format,
- requested_model.as_deref(),
+ served_model.as_deref(),
)?)
.into_response()),
LlmResponse::Stream(stream) => Ok(frame_stream(
- encode_stream(stream, target_format, requested_model)?,
+ encode_stream(stream, target_format, served_model)?,
target_format,
)
.into_response()),
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index c08834c26..b6ee689c6 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -844,6 +844,9 @@ async fn all_inbound_formats_run_libsy_and_return_the_caller_format() -> TestRes
.and_then(|value| value.to_str().ok()),
Some("model/a")
);
+ // The body names the model that answered, not the route id the caller
+ // addressed, so it agrees with the routing header above.
+ assert_eq!(response.json()?["model"], "model/a");
}
let calls = upstream.calls.lock().await;
@@ -874,6 +877,64 @@ async fn streaming_response_is_framed_for_the_inbound_api() -> TestResult {
Ok(())
}
+// SWITCH-922: every streaming codec must report the routed target, not the route
+// id the caller addressed — the route id is meaningless to anything reading the
+// trajectory (a Bench UI, a spend log, the client's own display).
+#[tokio::test]
+async fn streamed_response_model_names_the_served_model_not_the_route() -> TestResult {
+ let (_upstream, app) = test_app(&[(ROUTE_MODEL, &["model/a"])]).await?;
+
+ // Each case names the JSON pointer to the model on that format's first event.
+ let cases = [
+ (
+ "/v1/chat/completions",
+ json!({
+ "model": ROUTE_MODEL,
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": true
+ }),
+ vec!["model"],
+ ),
+ (
+ "/v1/messages",
+ json!({
+ "model": ROUTE_MODEL,
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hi"}],
+ "stream": true
+ }),
+ vec!["message", "model"],
+ ),
+ (
+ "/v1/responses",
+ json!({"model": ROUTE_MODEL, "input": "hi", "stream": true}),
+ vec!["response", "model"],
+ ),
+ ];
+
+ for (path, body, pointer) in cases {
+ let response = send(&app, "POST", path, Some(body)).await?;
+ assert_eq!(response.status, StatusCode::OK, "{path}");
+
+ let first = first_sse_event(response.text()?)
+ .ok_or_else(|| format!("{path} produced no SSE data frames"))?;
+ let model = pointer
+ .iter()
+ .try_fold(&first, |value, key| value.get(key))
+ .and_then(Value::as_str);
+ assert_eq!(model, Some("model/a"), "{path}");
+ }
+ Ok(())
+}
+
+// Returns the first `data:` frame of an SSE body as JSON, skipping `[DONE]`.
+fn first_sse_event(body: &str) -> Option<Value> {
+ body.lines()
+ .filter_map(|line| line.strip_prefix("data: "))
+ .filter(|data| *data != "[DONE]")
+ .find_map(|data| serde_json::from_str(data).ok())
+}
+
#[tokio::test]
async fn streaming_success_records_only_final_usage_and_one_latency() -> TestResult {
const MODEL: &str = "model/stream-success";
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index b2dbca718..78fdd2b54 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -6,8 +6,8 @@
use serde_json::{json, Map, Value};
use crate::codecs::stream::{
- record_source_identity, source_model_or_unknown, state_source_is, string_field, StreamCodec,
- StreamTranslationState,
+ record_source_identity, state_source_is, string_field, target_model_or_source_model,
+ StreamCodec, StreamTranslationState,
};
use crate::format::{FormatId, WireFormat};
use crate::llm::Usage;
@@ -285,7 +285,7 @@ fn openai_stream_chunk(
"id": openai_stream_id(state),
"object": "chat.completion.chunk",
"created": 0,
- "model": source_model_or_unknown(state),
+ "model": target_model_or_source_model(state),
"choices": [{
"index": 0,
"delta": delta,
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index 38cd73b91..a859432b7 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -27,7 +27,8 @@ pub struct StreamTranslationState {
pub model: Option<String>,
/// Message/response ID observed on the source provider stream.
pub message_id: Option<String>,
- /// Optional model name the target stream should expose to the client.
+ /// Model that served the call, exposed to the client in place of the id the
+ /// source stream reported. `None` falls back to [`Self::model`].
pub target_model: Option<String>,
/// Optional message/response ID the target stream should expose to the client.
pub target_message_id: Option<String>,
@@ -240,12 +241,7 @@ pub(crate) fn record_source_identity(
}
}
-// Returns the source model observed from the upstream stream.
-pub(crate) fn source_model_or_unknown(state: &StreamTranslationState) -> String {
- state.model.clone().unwrap_or_else(|| "unknown".to_string())
-}
-
-// Returns the target/client model when supplied, otherwise the upstream model.
+// Returns the served model when supplied, otherwise the upstream model.
pub(crate) fn target_model_or_source_model(state: &StreamTranslationState) -> String {
state
.target_model
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index d7486cf4a..19ffa82c5 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -48,16 +48,17 @@ pub fn decode_aggregated_response(body: &Value, wire_format: WireFormat) -> Resu
}
/// Encodes a buffered aggregate into `wire_format`'s JSON body, stamping
-/// `requested_model` over the upstream id so the caller sees the model it asked for.
+/// `served_model` over the encoded id so the caller sees which model answered.
+/// Passing `None` leaves the id the upstream reported.
pub fn encode_aggregated_response(
agg: &AggLlmResponse,
wire_format: WireFormat,
- requested_model: Option<&str>,
+ served_model: Option<&str>,
) -> Result<Value> {
let mut body = DEFAULT_TRANSLATION_ENGINE
.encode_response(wire_format, agg, &DEFAULT_TRANSLATION_POLICY)?
.body;
- if let (Some(model), Value::Object(object)) = (requested_model, &mut body) {
+ if let (Some(model), Value::Object(object)) = (served_model, &mut body) {
object.insert("model".to_string(), Value::String(model.to_string()));
}
Ok(body)
@@ -75,13 +76,14 @@ pub type RawEventStream = Pin<
/// Encodes a stream of IR chunks into a stream of target-format wire events.
///
-/// `requested_model` is exposed as the response model (via the stream state's
-/// `target_model`). The target stream codec is resolved once and reused per chunk;
-/// terminal events (`message_stop` / `response.completed`) come from `finish`.
+/// `served_model` is exposed as the response model (via the stream state's
+/// `target_model`); `None` falls back to the id observed on the source stream.
+/// The target stream codec is resolved once and reused per chunk; terminal events
+/// (`message_stop` / `response.completed`) come from `finish`.
pub fn encode_stream(
chunks: LlmResponseStream,
target: WireFormat,
- requested_model: Option<String>,
+ served_model: Option<String>,
) -> std::result::Result<RawEventStream, LlmClientError> {
// The target is always a built-in wire format, so this lookup cannot fail; a
// failure returns as an `Err` rather than a panic.
@@ -94,7 +96,7 @@ pub fn encode_stream(
let mut state = StreamTranslationState {
target: Some(target.into()),
- target_model: requested_model,
+ target_model: served_model,
..Default::default()
};
let mut chunks = chunks;
@@ -246,7 +248,7 @@ mod tests {
}
#[test]
- fn aggregated_response_round_trips_and_restamps_model() -> Result<(), BoxError> {
+ fn aggregated_response_round_trips_and_stamps_the_served_model() -> Result<(), BoxError> {
let body = json!({
"id": "1",
"model": "upstream",
@@ -260,10 +262,10 @@ mod tests {
let agg = decode_aggregated_response(&body, WireFormat::OpenAiChat)?;
assert_eq!(completion_text(&agg), "Hi there");
- // `requested_model` overrides the encoded model.
+ // `served_model` overrides the id the upstream reported.
let encoded =
- encode_aggregated_response(&agg, WireFormat::OpenAiChat, Some("client-facing"))?;
- assert_eq!(encoded["model"], "client-facing");
+ encode_aggregated_response(&agg, WireFormat::OpenAiChat, Some("served/model"))?;
+ assert_eq!(encoded["model"], "served/model");
assert_eq!(encoded["choices"][0]["message"]["content"], "Hi there");
Ok(())
}
@@ -304,6 +306,64 @@ mod tests {
Ok(())
}
+ // The served model must win over the id the source stream announced, so a
+ // routed response never reports the route the caller addressed.
+ #[test]
+ fn encode_stream_stamps_the_served_model_on_message_start() -> Result<(), BoxError> {
+ let chunks: LlmResponseStream = stream::iter(vec![
+ Ok(LlmResponseChunk::MessageStart {
+ id: Some("msg_1".to_string()),
+ model: Some("upstream/model".to_string()),
+ }),
+ Ok(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hi".to_string(),
+ }),
+ ])
+ .boxed();
+
+ let events = block_on(
+ encode_stream(
+ chunks,
+ WireFormat::AnthropicMessages,
+ Some("served/model".to_string()),
+ )?
+ .collect::<Vec<_>>(),
+ )
+ .into_iter()
+ .collect::<Result<Vec<Value>, BoxError>>()?;
+
+ assert_eq!(events[0]["type"], "message_start");
+ assert_eq!(events[0]["message"]["model"], "served/model");
+ Ok(())
+ }
+
+ // Without a served model the upstream id survives, so passthrough routes keep
+ // reporting whatever the provider announced.
+ #[test]
+ fn encode_stream_falls_back_to_the_source_model() -> Result<(), BoxError> {
+ let chunks: LlmResponseStream = stream::iter(vec![
+ Ok(LlmResponseChunk::MessageStart {
+ id: Some("msg_1".to_string()),
+ model: Some("upstream/model".to_string()),
+ }),
+ Ok(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hi".to_string(),
+ }),
+ ])
+ .boxed();
+
+ let events = block_on(
+ encode_stream(chunks, WireFormat::AnthropicMessages, None)?.collect::<Vec<_>>(),
+ )
+ .into_iter()
+ .collect::<Result<Vec<Value>, BoxError>>()?;
+
+ assert_eq!(events[0]["message"]["model"], "upstream/model");
+ Ok(())
+ }
+
#[test]
fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> {
let chunks: LlmResponseStream =
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 27bd9c411..81e5adbc0 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -91,13 +91,13 @@ fn anthropic_stream_usage_and_stop_translate_to_openai_chunks() -> TestResult {
Ok(())
}
-// Verifies Chat target streams expose upstream model identity, not the client request alias.
+// Verifies Chat target streams expose the served model while retaining source identity.
#[test]
-fn anthropic_to_openai_chat_uses_source_model_even_with_target_override() -> TestResult {
+fn anthropic_to_openai_chat_uses_served_model_without_losing_source_model() -> TestResult {
let engine = TranslationEngine::default();
let mut state =
StreamTranslationState::new(WireFormat::AnthropicMessages, WireFormat::OpenAiChat);
- state.target_model = Some("gpt-client".to_string());
+ state.target_model = Some("served-model".to_string());
let start = json!({
"type": "message_start",
"message": {
@@ -127,18 +127,18 @@ fn anthropic_to_openai_chat_uses_source_model_even_with_target_override() -> Tes
)?;
assert_eq!(state.model.as_deref(), Some("claude-upstream"));
- assert_eq!(state.target_model.as_deref(), Some("gpt-client"));
- assert_eq!(events[0]["model"], "claude-upstream");
+ assert_eq!(state.target_model.as_deref(), Some("served-model"));
+ assert_eq!(events[0]["model"], "served-model");
Ok(())
}
-// Verifies Anthropic target streams can expose a client model without losing source identity.
+// Verifies Anthropic target streams expose the served model without losing source identity.
#[test]
-fn responses_to_anthropic_uses_target_model_without_losing_source_model() -> TestResult {
+fn responses_to_anthropic_uses_served_model_without_losing_source_model() -> TestResult {
let engine = TranslationEngine::default();
let mut state =
StreamTranslationState::new(WireFormat::OpenAiResponses, WireFormat::AnthropicMessages);
- state.target_model = Some("claude-client".to_string());
+ state.target_model = Some("served-model".to_string());
let created = json!({
"type": "response.created",
"response": {"id": "resp_1", "model": "responses-upstream"}
@@ -152,18 +152,18 @@ fn responses_to_anthropic_uses_target_model_without_losing_source_model() -> Tes
)?;
assert_eq!(state.model.as_deref(), Some("responses-upstream"));
- assert_eq!(state.target_model.as_deref(), Some("claude-client"));
- assert_eq!(events[0]["message"]["model"], "claude-client");
+ assert_eq!(state.target_model.as_deref(), Some("served-model"));
+ assert_eq!(events[0]["message"]["model"], "served-model");
Ok(())
}
-// Verifies Responses target streams can expose a client model while retaining source identity.
+// Verifies Responses target streams expose the served model while retaining source identity.
#[test]
-fn openai_chat_to_responses_uses_target_model_without_losing_source_model() -> TestResult {
+fn openai_chat_to_responses_uses_served_model_without_losing_source_model() -> TestResult {
let engine = TranslationEngine::default();
let mut state =
StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiResponses);
- state.target_model = Some("responses-client".to_string());
+ state.target_model = Some("served-model".to_string());
let chunk = json!({
"id": "chatcmpl-test",
"object": "chat.completion.chunk",
@@ -183,8 +183,8 @@ fn openai_chat_to_responses_uses_target_model_without_losing_source_model() -> T
)?;
assert_eq!(state.model.as_deref(), Some("gpt-upstream"));
- assert_eq!(state.target_model.as_deref(), Some("responses-client"));
- assert_eq!(events[0]["response"]["model"], "responses-client");
+ assert_eq!(state.target_model.as_deref(), Some("served-model"));
+ assert_eq!(events[0]["response"]["model"], "served-model");
Ok(())
}
diff --git a/switchyard/lib/processors/format_translate.py b/switchyard/lib/processors/format_translate.py
index 126bcf11e..c4c2d5b23 100644
--- a/switchyard/lib/processors/format_translate.py
+++ b/switchyard/lib/processors/format_translate.py
@@ -195,9 +195,9 @@ class FormatTranslateResponseProcessor:
is already in the original format or when ``CTX_ORIGINAL_FORMAT``
is absent.
- Streaming responses use the original request body captured in
- context to preserve target-format stream metadata such as model
- names while translating lazily.
+ Streaming responses take the response model from the routing context so
+ the translated stream names the model that actually served the call,
+ while still translating lazily.
"""
async def process(self, ctx: ProxyContext, response: ChatResponse) -> ChatResponse:
@@ -221,7 +221,7 @@ class FormatTranslateResponseProcessor:
return TranslationEngine().response_to(
ChatRequestType.OPENAI_RESPONSES,
response,
- original_body=_original_body(ctx),
+ served_model=_served_model(ctx),
)
raise NotImplementedError(
f"Unsupported original format for response translation: {original!r}",
@@ -238,7 +238,7 @@ def _translate_streaming_response(
return TranslationEngine().response_to(
original,
response,
- original_body=_original_body(ctx),
+ served_model=_served_model(ctx),
)
@@ -255,11 +255,10 @@ def _request_for_original_format(
raise NotImplementedError(f"Unsupported original format: {original!r}")
-def _original_body(ctx: ProxyContext) -> dict[str, object]:
- body = ctx.metadata.get(CTX_ORIGINAL_REQUEST)
- if isinstance(body, Mapping):
- return dict(body)
- return {}
+def _served_model(ctx: ProxyContext) -> str | None:
+ """Return the model the backend actually called, or ``None`` when unknown."""
+ model = ctx.selected_model or ctx.metadata.get(CTX_PROXY_ACTUAL_MODEL)
+ return model if isinstance(model, str) and model else None
def _matches_format(response: ChatResponse, target: ChatRequestType | str) -> bool:
diff --git a/switchyard_rust/translation.py b/switchyard_rust/translation.py
index 07259315f..b7b5c9843 100644
--- a/switchyard_rust/translation.py
+++ b/switchyard_rust/translation.py
@@ -145,7 +145,7 @@ class TranslationEngine:
target: str | ChatRequestType,
response: ChatResponse,
*,
- original_body: Mapping[str, Any] | None = None,
+ served_model: str | None = None,
) -> ChatResponse:
"""Return a ChatResponse wrapper in the target wire format."""
source = _response_format(response)
@@ -155,7 +155,7 @@ class TranslationEngine:
if _is_streaming_response(response):
return _wrap_streaming_response(
target_format,
- self._translate_response_stream(source, target_format, response, original_body),
+ self._translate_response_stream(source, target_format, response, served_model),
)
return _wrap_response(
target_format,
@@ -166,10 +166,12 @@ class TranslationEngine:
self,
request: ChatRequest,
response: ChatResponse,
+ *,
+ served_model: str | None = None,
) -> TranslatedResponse:
"""Translate a backend response to the original client's wire format."""
if _is_streaming_response(response):
- return self.stream_for_request(request, response)
+ return self.stream_for_request(request, response, served_model=served_model)
source = _response_format(response)
target = _request_format(request)
if source == target:
@@ -183,6 +185,8 @@ class TranslationEngine:
self,
request: ChatRequest,
response: ChatResponse,
+ *,
+ served_model: str | None = None,
) -> TranslatedStream:
"""Translate a backend stream to the original client's stream contract."""
source = _response_format(response)
@@ -191,7 +195,7 @@ class TranslationEngine:
return cast("TranslatedStream", _response_stream(response))
return cast(
"TranslatedStream",
- self._translate_response_stream(source, target, response, getattr(request, "body", None)),
+ self._translate_response_stream(source, target, response, served_model),
)
async def translate(
@@ -200,9 +204,13 @@ class TranslationEngine:
request: ChatRequest,
response: ChatResponse,
) -> TranslatedResponse:
- """Implement Switchyard's terminal TranslationEngine role."""
- _ = ctx
- return self.response_for_request(request, response)
+ """Implement Switchyard's terminal TranslationEngine role.
+
+ The served model comes from the routing context, never from a request
+ body: routers rewrite the model on their own copy of the request, so the
+ request reaching this role can still name the route the client addressed.
+ """
+ return self.response_for_request(request, response, served_model=ctx.selected_model)
async def translate_stream(
self,
@@ -244,13 +252,13 @@ class TranslationEngine:
source: _NativeFormat,
target: _NativeFormat,
response: ChatResponse,
- original_body: Mapping[str, Any] | object | None,
+ served_model: str | None,
) -> AsyncGenerator[Any, None]:
return self.translate_stream(
source,
target,
_response_stream(response),
- model=_model_from_body(original_body),
+ model=served_model,
output=_stream_wire_output_for_target(target),
)
@@ -369,14 +377,6 @@ def _stream_wire_output_for_target(target: _NativeFormat) -> _StreamOutput:
return "objects"
-def _model_from_body(body: Mapping[str, Any] | object | None) -> str | None:
- if isinstance(body, Mapping):
- model = body.get("model")
- if isinstance(model, str) and model:
- return model
- return None
-
-
def _response_body(response: ChatResponse) -> Any:
return response.body
diff --git a/tests/test_deterministic_routing_profile.py b/tests/test_deterministic_routing_profile.py
index 331b3bb21..eda118192 100644
--- a/tests/test_deterministic_routing_profile.py
+++ b/tests/test_deterministic_routing_profile.py
@@ -8,6 +8,7 @@ from __future__ import annotations
from typing import Any
import pytest
+from openai.types.chat import ChatCompletionChunk
from pydantic import ValidationError
from switchyard.lib.backends.deterministic_routing_llm_backend import (
@@ -15,6 +16,7 @@ from switchyard.lib.backends.deterministic_routing_llm_backend import (
DeterministicRoutingLLMBackend,
)
from switchyard.lib.backends.llm_target import BackendFormat, LlmTarget
+from switchyard.lib.chat_response.openai_chat import ResponseStream
from switchyard.lib.processors.llm_classifier import (
LLMClassifierRequestProcessor,
SignalTierSelectorRequestProcessor,
@@ -617,3 +619,56 @@ async def test_overflow_reroutes_to_custom_strong_id_through_full_profile() -> N
assert cheap.calls == 1
assert frontier.calls == 1
assert response.body["choices"][0]["message"]["content"] == "ok"
+
+
+class _StreamingTier(LLMBackend):
+ """Stub tier backend that answers with an OpenAI-format stream."""
+
+ @property
+ def supported_request_types(self) -> list[ChatRequestType]:
+ return [ChatRequestType.OPENAI_CHAT]
+
+ async def call(self, ctx: ProxyContext, request: ChatRequest) -> ChatResponse:
+ async def chunks() -> Any:
+ yield ChatCompletionChunk.model_validate({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "created": 1700000000,
+ "model": request.model,
+ "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}],
+ })
+
+ return ChatResponse.openai_stream(ResponseStream(chunks()))
+
+
+async def test_streamed_response_names_the_tier_model_not_the_route() -> None:
+ """A routed, translated stream reports the tier's model, not the route key.
+
+ Regression test for SWITCH-922: the terminal translator used to take the
+ response model from the request body. Every Rust request processor hands
+ the chain a *clone*, so the backend's ``set_model`` never reached the
+ translator and routed turns were labelled with the route id the client
+ addressed — literally ``switchyard`` for the benchmark bundles.
+ """
+ config = _config(enable_stats=True)
+ profile = DeterministicRoutingProfileConfig.from_config(config).build()
+ profile._request_processors = (_StampTier("weak"),)
+
+ # Stats must stay on: its Rust request processor is what hands the chain a
+ # clone, which is the condition this regression test exists for. Swap the
+ # tier backends only after the runtime wiring, which cannot wrap stubs.
+ runtime = profile.with_runtime_components(enable_stats=config.enable_stats)
+ backend = profile._backend
+ assert isinstance(backend, DeterministicRoutingLLMBackend)
+ backend._backends = {"strong": _StreamingTier(), "weak": _StreamingTier()}
+
+ switchyard = ProfileSwitchyard(runtime)
+ request = ChatRequest.anthropic({
+ "model": "switchyard",
+ "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hi"}],
+ })
+
+ events = [event async for event in await switchyard.call(request)]
+
+ assert events[0]["message"]["model"] == config.weak.model
diff --git a/tests/test_format_translate_processor.py b/tests/test_format_translate_processor.py
index 05aecf139..1686723e1 100644
--- a/tests/test_format_translate_processor.py
+++ b/tests/test_format_translate_processor.py
@@ -311,6 +311,7 @@ async def test_streaming_openai_response_translates_back_to_anthropic() -> None:
CTX_ORIGINAL_REQUEST: {"model": "claude-client"},
}
)
+ ctx.selected_model = "served/model"
response = ChatResponse.openai_stream(
ResponseStream(
_aiter(
@@ -331,7 +332,7 @@ async def test_streaming_openai_response_translates_back_to_anthropic() -> None:
for event in events
if event["type"] == "content_block_delta" and event["delta"]["type"] == "text_delta"
)
- assert events[0]["message"]["model"] == "claude-client"
+ assert events[0]["message"]["model"] == "served/model"
assert text == "hello"
assert [event["type"] for event in events].count("content_block_start") == 1
@@ -364,6 +365,7 @@ async def test_streaming_openai_response_translates_back_to_responses() -> None:
CTX_ORIGINAL_REQUEST: {"model": "responses-client", "input": "hi"},
}
)
+ ctx.selected_model = "served/model"
response = ChatResponse.openai_stream(
ResponseStream(
_aiter(
@@ -380,7 +382,7 @@ async def test_streaming_openai_response_translates_back_to_responses() -> None:
assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_STREAM)
frames = await _collect(result.stream)
payloads = [_frame_data(frame) for frame in frames]
- assert payloads[0]["response"]["model"] == "responses-client"
+ assert payloads[0]["response"]["model"] == "served/model"
assert (
"".join(
payload["delta"]
@@ -549,6 +551,7 @@ async def test_streaming_responses_response_translates_back_to_anthropic() -> No
CTX_ORIGINAL_REQUEST: {"model": "claude-client"},
}
)
+ ctx.selected_model = "served/model"
response = ChatResponse.openai_responses_stream(
ResponsesApiStream(_aiter(_responses_events())),
)
@@ -562,7 +565,7 @@ async def test_streaming_responses_response_translates_back_to_anthropic() -> No
for event in events
if event["type"] == "content_block_delta" and event["delta"]["type"] == "text_delta"
)
- assert events[0]["message"]["model"] == "claude-client"
+ assert events[0]["message"]["model"] == "served/model"
assert text == "hello"
diff --git a/tests/test_response_translation_engine.py b/tests/test_response_translation_engine.py
index fd41b29ab..6ef605757 100644
--- a/tests/test_response_translation_engine.py
+++ b/tests/test_response_translation_engine.py
@@ -16,14 +16,17 @@ from openai.types.chat.chat_completion_message import ChatCompletionMessage
from openai.types.completion_usage import CompletionUsage
from openai.types.responses import Response as OpenAIResponse
+from switchyard.lib.chat_response.anthropic import AnthropicResponseStream
from switchyard.lib.chat_response.openai_chat import ResponseStream
from switchyard_rust.core import (
+ ChatRequest,
ChatRequestType,
ChatResponse,
ChatResponseType,
+ ProxyContext,
response_type_matches,
)
-from switchyard_rust.translation import TranslationEngine
+from switchyard_rust.translation import TranslationEngine, sse_frame_payloads
E = TranslationEngine()
@@ -177,11 +180,7 @@ class TestToResponses:
def test_responses_passthrough(self):
"""Responses API completion responses pass through unchanged."""
resp = ChatResponse.openai_responses_completion(_make_responses_api_response())
- result = E.response_to(
- ChatRequestType.OPENAI_RESPONSES,
- resp,
- original_body={"model": "gpt-4o"},
- )
+ result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp)
assert result is resp
def test_responses_streaming_passthrough(self):
@@ -189,21 +188,13 @@ class TestToResponses:
from switchyard.lib.chat_response.openai_responses import ResponsesApiStream
resp = ChatResponse.openai_responses_stream(ResponsesApiStream(_async_iter([])))
- result = E.response_to(
- ChatRequestType.OPENAI_RESPONSES,
- resp,
- original_body={"model": "gpt-4o"},
- )
+ result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp)
assert result is resp
def test_openai_to_responses(self):
"""OpenAI completion responses are converted to Responses API responses."""
resp = ChatResponse.openai_completion(_make_completion(content="Hello!"))
- result = E.response_to(
- ChatRequestType.OPENAI_RESPONSES,
- resp,
- original_body={"model": "gpt-4o", "input": "Hi"},
- )
+ result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp)
assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION)
body = result.body
@@ -215,11 +206,7 @@ class TestToResponses:
def test_anthropic_to_responses(self):
"""Anthropic responses are converted to Responses API responses."""
resp = ChatResponse.anthropic_completion(_make_anthropic_message())
- result = E.response_to(
- ChatRequestType.OPENAI_RESPONSES,
- resp,
- original_body={"model": "gpt-4o"},
- )
+ result = E.response_to(ChatRequestType.OPENAI_RESPONSES, resp)
assert response_type_matches(result, ChatResponseType.OPENAI_RESPONSES_COMPLETION)
assert result.body["output"][0]["content"][0]["text"] == "Hi there"
@@ -271,6 +258,134 @@ def _find_event(events: list[dict], event_type: str) -> dict:
return next(ev for ev in events if ev["type"] == event_type)
+# =========================================================================
+# terminal translate() role — served-model stamping (SWITCH-922)
+# =========================================================================
+
+
+_ROUTE_ID = "switchyard"
+_SERVED_MODEL = "served/model"
+
+
+def _routed_ctx(served_model: str | None = _SERVED_MODEL) -> ProxyContext:
+ """A context shaped like one a router leaves behind for the terminal role."""
+ ctx = ProxyContext()
+ ctx.selected_model = served_model
+ return ctx
+
+
+class TestTranslateStampsTheServedModel:
+ """The client-visible ``model`` must name the model that answered.
+
+ Routers rewrite the model on their own copy of the request, so the request
+ reaching the terminal role still names the route the client addressed
+ (``switchyard`` for the benchmark bundles). Reading it from there labelled
+ every routed turn with the route id.
+ """
+
+ @pytest.mark.asyncio
+ async def test_anthropic_stream_reports_the_served_model(self):
+ request = ChatRequest.anthropic(
+ {"model": _ROUTE_ID, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
+ )
+ response = ChatResponse.openai_stream(ResponseStream(_async_iter([_chunk(content="hi")])))
+
+ events = [ev async for ev in await E.translate(_routed_ctx(), request, response)]
+
+ assert events[0]["message"]["model"] == _SERVED_MODEL
+
+ @pytest.mark.asyncio
+ async def test_responses_stream_reports_the_served_model(self):
+ request = ChatRequest.openai_responses({"model": _ROUTE_ID, "input": "hi"})
+ response = ChatResponse.openai_stream(ResponseStream(_async_iter([_chunk(content="hi")])))
+
+ frames = [frame async for frame in await E.translate(_routed_ctx(), request, response)]
+ payloads = [
+ payload for frame in frames for payload in sse_frame_payloads(frame)
+ ]
+
+ assert payloads[0]["response"]["model"] == _SERVED_MODEL
+
+ @pytest.mark.asyncio
+ async def test_openai_chat_stream_reports_the_served_model(self):
+ request = ChatRequest.openai_chat(
+ {"model": _ROUTE_ID, "messages": [{"role": "user", "content": "hi"}]}
+ )
+ response = ChatResponse.anthropic_stream(
+ AnthropicResponseStream(
+ _async_iter(
+ [
+ {
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "model": "upstream/model",
+ "role": "assistant",
+ "content": [],
+ },
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "text_delta", "text": "hi"},
+ },
+ ]
+ )
+ )
+ )
+
+ chunks = [chunk async for chunk in await E.translate(_routed_ctx(), request, response)]
+
+ # The OpenAI-chat target yields SDK chunk objects, not dicts.
+ assert chunks[0].model == _SERVED_MODEL
+
+ @pytest.mark.asyncio
+ async def test_buffered_response_reports_the_served_model(self):
+ request = ChatRequest.anthropic(
+ {"model": _ROUTE_ID, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
+ )
+ response = ChatResponse.openai_completion(_make_completion(model=_SERVED_MODEL))
+
+ result = await E.translate(_routed_ctx(), request, response)
+
+ assert result["model"] == _SERVED_MODEL
+
+ @pytest.mark.asyncio
+ async def test_stream_falls_back_to_the_upstream_model_without_a_selection(self):
+ """Passthrough routes keep reporting whatever the provider announced."""
+ request = ChatRequest.openai_chat(
+ {"model": _ROUTE_ID, "messages": [{"role": "user", "content": "hi"}]}
+ )
+ response = ChatResponse.anthropic_stream(
+ AnthropicResponseStream(
+ _async_iter(
+ [
+ {
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "model": "upstream/model",
+ "role": "assistant",
+ "content": [],
+ },
+ },
+ {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "text_delta", "text": "hi"},
+ },
+ ]
+ )
+ )
+ )
+
+ chunks = [
+ chunk async for chunk in await E.translate(_routed_ctx(None), request, response)
+ ]
+
+ assert chunks[0].model == "upstream/model"
+
+
class TestStreamOpenAIToAnthropicUsage:
@pytest.mark.asyncio
async def test_backend_usage_preferred_over_heuristic(self):