Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(translation): move stream translation into TranslationEngine (#1220)

David Gardner4f23dedaa822
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index f901d875e..135fb645c 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -21,6 +21,20 @@ impl StreamCodec for AnthropicMessagesStreamCodec {
WireFormat::AnthropicMessages.into()
}
+ fn is_terminal_event(&self, event: &Value) -> bool {
+ event.get("type").and_then(Value::as_str) == Some("message_stop")
+ }
+
+ fn sse_done_is_terminal(&self) -> bool {
+ false
+ }
+
+ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {
+ if let Some(message) = event.get_mut("message").and_then(Value::as_object_mut) {
+ message.insert("model".to_string(), Value::String(served_model.to_string()));
+ }
+ }
+
fn decode_event(
&self,
state: &mut StreamTranslationState,
@@ -745,3 +759,33 @@ fn tool_input_delta(value: &Value) -> Option<String> {
_ => None,
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn recognizes_stream_completion() {
+ let codec = AnthropicMessagesStreamCodec;
+ assert!(!codec.sse_done_is_terminal());
+ assert!(codec.is_terminal_event(&json!({"type": "message_stop"})));
+ assert!(!codec.is_terminal_event(&json!({"type": "message_delta"})));
+ }
+
+ #[test]
+ fn anthropic_replay_stamps_the_served_model_inside_message() {
+ let mut event = json!({
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "model": "provider/model",
+ "provider_extension": true,
+ },
+ });
+
+ AnthropicMessagesStreamCodec.stamp_streamed_response_model(&mut event, "served/model");
+
+ assert_eq!(event["message"]["model"], "served/model");
+ assert_eq!(event["message"]["provider_extension"], true);
+ }
+}
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index 02cec4b6e..bc4da4ea2 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -22,6 +22,30 @@ impl StreamCodec for OpenAiChatStreamCodec {
WireFormat::OpenAiChat.into()
}
+ fn is_terminal_event(&self, event: &Value) -> bool {
+ event
+ .get("choices")
+ .and_then(Value::as_array)
+ .into_iter()
+ .flatten()
+ .any(|choice| {
+ choice
+ .get("finish_reason")
+ .and_then(Value::as_str)
+ .is_some()
+ })
+ }
+
+ fn sse_done_is_terminal(&self) -> bool {
+ true
+ }
+
+ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {
+ if let Some(event) = event.as_object_mut() {
+ event.insert("model".to_string(), Value::String(served_model.to_string()));
+ }
+ }
+
fn decode_event(
&self,
state: &mut StreamTranslationState,
@@ -444,3 +468,29 @@ fn openai_finish_reason(reason: Option<&str>) -> String {
Some(other) => other.to_string(),
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn recognizes_stream_completion() {
+ let codec = OpenAiChatStreamCodec;
+ assert!(codec.sse_done_is_terminal());
+ assert!(codec.is_terminal_event(&json!({"choices": [{"finish_reason": "stop"}]})));
+ assert!(!codec.is_terminal_event(&json!({"choices": [{"finish_reason": null}]})));
+ }
+
+ #[test]
+ fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {
+ let mut event = json!({
+ "choices": [{"delta": {"content": "Hello"}}],
+ "system_fingerprint": "fp_provider_specific",
+ });
+
+ OpenAiChatStreamCodec.stamp_streamed_response_model(&mut event, "served/model");
+
+ assert_eq!(event["model"], "served/model");
+ assert_eq!(event["system_fingerprint"], "fp_provider_specific");
+ }
+}
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index b726bea71..8d4a5c3c7 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -27,6 +27,26 @@ impl StreamCodec for OpenAiResponsesStreamCodec {
WireFormat::OpenAiResponses.into()
}
+ fn is_terminal_event(&self, event: &Value) -> bool {
+ matches!(
+ event
+ .get("type")
+ .or_else(|| event.get("event"))
+ .and_then(Value::as_str),
+ Some("response.completed" | "response.incomplete" | "response.failed")
+ )
+ }
+
+ fn sse_done_is_terminal(&self) -> bool {
+ true
+ }
+
+ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {
+ if let Some(response) = event.get_mut("response").and_then(Value::as_object_mut) {
+ response.insert("model".to_string(), Value::String(served_model.to_string()));
+ }
+ }
+
fn decode_event(
&self,
state: &mut StreamTranslationState,
@@ -1226,3 +1246,40 @@ fn responses_id(state: &StreamTranslationState) -> String {
.unwrap_or("resp_switchyard")
.to_string()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn recognizes_stream_completion() {
+ let codec = OpenAiResponsesStreamCodec;
+ assert!(codec.sse_done_is_terminal());
+ for kind in [
+ "response.completed",
+ "response.incomplete",
+ "response.failed",
+ ] {
+ assert!(codec.is_terminal_event(&json!({"type": kind})));
+ assert!(codec.is_terminal_event(&json!({"event": kind})));
+ }
+ assert!(!codec.is_terminal_event(&json!({"type": "response.created"})));
+ }
+
+ #[test]
+ fn responses_replay_stamps_the_served_model_inside_response() {
+ let mut event = json!({
+ "type": "response.created",
+ "response": {
+ "id": "resp_1",
+ "model": "provider/model",
+ "provider_extension": true,
+ },
+ });
+
+ OpenAiResponsesStreamCodec.stamp_streamed_response_model(&mut event, "served/model");
+
+ assert_eq!(event["response"]["model"], "served/model");
+ assert_eq!(event["response"]["provider_extension"], true);
+ }
+}
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index 04f3b0f29..145679abf 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -169,6 +169,20 @@ pub trait StreamCodec: Send + Sync {
/// Returns the stream format handled by this codec.
fn format(&self) -> FormatId;
+ /// Returns whether a JSON event establishes stream completion.
+ fn is_terminal_event(&self, event: &Value) -> bool;
+
+ /// Returns whether the SSE `[DONE]` marker establishes stream completion.
+ ///
+ /// The marker ends consumption even when this returns `false`; a terminal JSON event
+ /// must then have been observed to avoid a premature-EOF error.
+ fn sse_done_is_terminal(&self) -> bool;
+
+ /// Overrides the model reported by an encoded or preserved event.
+ ///
+ /// Called only when a served model was supplied, including for events emitted by `finish`.
+ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str);
+
/// Decodes one provider event into zero or more neutral events.
fn decode_event(
&self,
diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs
index ae2020a87..c0865313b 100644
--- a/crates/switchyard-translation/src/engine.rs
+++ b/crates/switchyard-translation/src/engine.rs
@@ -1,14 +1,18 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Registry-backed translation engine for buffered requests and responses.
+//! Registry-backed translation engine for buffered and streaming translation.
use std::collections::BTreeMap;
+use std::pin::Pin;
use std::sync::Arc;
+use async_stream::try_stream;
+use futures::io::AsyncBufReadExt;
+use futures::{Stream, StreamExt, TryStreamExt};
use serde_json::Value;
+use switchyard_protocol::LlmClientError;
-use crate::LlmResponseStreamEvent;
use crate::codecs::FormatCodec;
use crate::codecs::anthropic::AnthropicMessagesCodec;
use crate::codecs::openai_chat::OpenAiChatCodec;
@@ -21,6 +25,14 @@ use crate::error::{Result, TranslationError};
use crate::format::FormatId;
use crate::llm::{AggLlmResponse, LlmRequest, ProviderExtensions};
use crate::policy::TranslationPolicy;
+use crate::sse;
+use crate::{LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError};
+
+/// A stream of wire-format event objects in one format — the unframed body of an
+/// SSE response. The serving layer frames each `Value` (e.g. as an SSE
+/// `data:`/`event:` block).
+pub type RawEventStream =
+ Pin<Box<dyn Stream<Item = std::result::Result<Value, LlmStreamError>> + Send>>;
/// Encoded translation result with any diagnostics emitted along the way.
#[derive(Debug)]
@@ -333,6 +345,238 @@ impl TranslationEngine {
let target_codec = self.stream_registry.codec(target)?;
Ok(target_codec.finish(state))
}
+
+ /// Encodes a stream of IR chunks into a stream of target-format wire events.
+ ///
+ /// `served_model` is exposed as the response model (via the stream state's
+ /// `target_model`); `None` falls back to the id observed on the source stream.
+ /// The target stream codec is resolved once and reused per chunk; terminal events
+ /// (`message_stop` / `response.completed`) come from `finish`.
+ pub fn encode_stream(
+ &self,
+ target: impl Into<FormatId>,
+ chunks: LlmResponseStream,
+ served_model: Option<String>,
+ ) -> std::result::Result<RawEventStream, LlmClientError> {
+ self.encode_stream_with_extensions(
+ target,
+ chunks,
+ served_model,
+ &ProviderExtensions::default(),
+ )
+ }
+
+ /// Encodes a response stream, honouring request extensions.
+ ///
+ /// Identical to [`Self::encode_stream`] except that Codex tool namespaces recorded on
+ /// the request are restored on each encoded event.
+ pub fn encode_stream_with_extensions(
+ &self,
+ target: impl Into<FormatId>,
+ chunks: LlmResponseStream,
+ served_model: Option<String>,
+ request_extensions: &ProviderExtensions,
+ ) -> std::result::Result<RawEventStream, LlmClientError> {
+ let origins = crate::codex_namespaces::qualified_tool_origins(request_extensions);
+ let custom_tools = crate::codex_custom_tools::custom_tool_names(request_extensions);
+ let mut custom_state = crate::codex_custom_tools::CustomToolCallStreamState::default();
+ let target_format: FormatId = target.into();
+ let codec = self
+ .stream_registry
+ .codec(target_format.clone())
+ // Currently the only error is that the codec is missing, which is Configuration
+ .map_err(|err| LlmClientError::Configuration {
+ message: err.to_string(),
+ })?;
+
+ let served_model_for_events = served_model.clone();
+ let mut state = StreamTranslationState {
+ target: Some(target_format.clone()),
+ target_model: served_model,
+ ..Default::default()
+ };
+ let mut chunks = chunks;
+
+ let events = try_stream! {
+ while let Some(item) = chunks.next().await {
+ let event = item.map_err(LlmStreamError::Client)?;
+ let mut encoded =
+ encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event);
+ for value in &mut encoded {
+ if let Some(model) = served_model_for_events.as_deref() {
+ codec.stamp_streamed_response_model(value, model);
+ }
+ crate::codex_namespaces::restore_qualified_tool_names(value, &origins);
+ }
+ // Argument deltas for a freeform tool cannot be expressed on the wire; the
+ // rewritten completed item carries the input instead.
+ let mut encoded: Vec<Value> = encoded
+ .into_iter()
+ .filter_map(|mut value| {
+ crate::codex_custom_tools::restore_custom_tool_calls_in_event(
+ &mut value,
+ &custom_tools,
+ &mut custom_state,
+ )
+ .then_some(value)
+ })
+ .collect();
+ let terminal = encoded.pop();
+ for value in encoded {
+ yield value;
+ }
+ match (state.errored, terminal) {
+ (false, Some(value)) => yield value,
+ (true, Some(value)) => Err(LlmStreamError::Upstream(value))?,
+ (true, None) => Err(LlmStreamError::Client(LlmClientError::ResponseTranslation(
+ "stream ended after an error without a terminal event".to_string(),
+ )))?,
+ _ => ()
+ }
+ }
+ for mut value in codec.finish(&mut state) {
+ if let Some(model) = served_model_for_events.as_deref() {
+ codec.stamp_streamed_response_model(&mut value, model);
+ }
+ crate::codex_namespaces::restore_qualified_tool_names(&mut value, &origins);
+ if crate::codex_custom_tools::restore_custom_tool_calls_in_event(
+ &mut value,
+ &custom_tools,
+ &mut custom_state,
+ ) {
+ yield value;
+ }
+ }
+ };
+
+ Ok(Box::pin(events))
+ }
+
+ /// Decodes provider SSE bytes into normalized stream events.
+ ///
+ /// Operates on raw bytes, not any HTTP client type: the caller adapts its
+ /// transport's body stream into `Stream<Item = Result<Vec<u8>, _>>`. Frames are
+ /// buffered across chunks (a partial frame waits for its boundary); the source
+ /// stream codec is resolved once and reused for every frame.
+ ///
+ /// The decoder tracks the source format's protocol-specific terminal event. If
+ /// EOF arrives without that required event, the stream yields a deferred
+ /// [`LlmClientError::ResponseTranslation`] error after any valid decoded events.
+ pub fn decode_stream<S>(
+ &self,
+ source: impl Into<FormatId>,
+ bytes: S,
+ ) -> std::result::Result<LlmResponseStream, LlmClientError>
+ where
+ S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,
+ {
+ let marker = sse::done_marker();
+ let source_format: FormatId = source.into();
+ let codec = self
+ .stream_registry
+ .codec(source_format.clone())
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ // Adapt the byte-chunk stream into an async line reader. The BufReader
+ // reassembles data split across network chunks (including multi-byte UTF-8),
+ // and `lines()` yields one SSE field line at a time. The stream is boxed to
+ // an `io::Error` item so `into_async_read`'s error bound resolves cleanly. The
+ // source error is boxed intact rather than stringified, so
+ // `llm_client_error_from_io` can recover its original variant on the way out.
+ let io_bytes: Pin<Box<dyn Stream<Item = std::io::Result<Vec<u8>>> + Send>> =
+ Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));
+ let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();
+
+ let mut state = StreamTranslationState {
+ source: Some(source_format.clone()),
+ ..StreamTranslationState::default()
+ };
+ let mut frame = String::new();
+ let mut saw_terminal = false;
+ let mut saw_error = false;
+ let stream = Box::pin(try_stream! {
+ futures::pin_mut!(lines);
+ while let Some(line) = lines.next().await {
+ let line = line.map_err(llm_client_error_from_io)?;
+ // A blank line (allowing a bare CR for CRLF streams) ends the frame.
+ if line.trim_end().is_empty() {
+ let parsed = sse::parse_json_sse_frame(&frame, marker)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ frame.clear();
+ match parsed {
+ sse::SseFrame::Empty => {}
+ sse::SseFrame::Done => {
+ saw_terminal |= codec.sse_done_is_terminal();
+ break;
+ }
+ sse::SseFrame::Data(value) => {
+ saw_terminal |= codec.is_terminal_event(&value);
+ let normalized = codec.decode_event(&mut state, &value);
+ saw_error |= normalized.iter().any(|chunk| matches!(
+ chunk,
+ LlmResponseChunk::DecodeError { .. }
+ | LlmResponseChunk::StreamError { .. }
+ ));
+ yield LlmResponseStreamEvent::preserved(
+ source_format.clone(),
+ value,
+ normalized,
+ );
+ }
+ }
+ } else {
+ frame.push_str(&line);
+ frame.push('\n');
+ }
+ }
+
+ // A non-standard upstream might omit the final blank line; parse a trailing
+ // complete frame instead of losing its last chunk.
+ #[allow(clippy::collapsible_if)]
+ if !frame.trim_end().is_empty() {
+ let parsed = sse::parse_json_sse_frame(&frame, marker)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ match parsed {
+ sse::SseFrame::Done => {
+ saw_terminal |= codec.sse_done_is_terminal();
+ }
+ sse::SseFrame::Data(value) => {
+ saw_terminal |= codec.is_terminal_event(&value);
+ let normalized = codec.decode_event(&mut state, &value);
+ saw_error |= normalized.iter().any(|chunk| matches!(
+ chunk,
+ LlmResponseChunk::DecodeError { .. }
+ | LlmResponseChunk::StreamError { .. }
+ ));
+ yield LlmResponseStreamEvent::preserved(source_format.clone(), value, normalized);
+ }
+ sse::SseFrame::Empty => {}
+ }
+ }
+
+ if !saw_terminal && !saw_error {
+ Err(LlmClientError::ResponseTranslation(format!(
+ "{source_format} stream ended before a terminal event"
+ )))?;
+ }
+ });
+ Ok(stream)
+ }
+}
+
+// Recover transport errors wrapped for `AsyncRead`; other reader failures are
+// invalid upstream responses.
+fn llm_client_error_from_io(error: std::io::Error) -> LlmClientError {
+ let kind = error.kind();
+ let message = error.to_string();
+ match error.into_inner() {
+ Some(source) => match source.downcast::<LlmClientError>() {
+ Ok(error) => *error,
+ Err(source) => LlmClientError::InvalidResponse { source },
+ },
+ None => LlmClientError::InvalidResponse {
+ source: Box::new(std::io::Error::new(kind, message)),
+ },
+ }
}
// Attaches source and target formats to every diagnostic emitted across both passes.
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index 95ebb75e7..ef6b39104 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -6,21 +6,15 @@
//! streamed response — so callers can translate without threading an engine and
//! policy through every call.
-use std::pin::Pin;
use std::sync::LazyLock;
-use async_stream::try_stream;
-use futures::io::AsyncBufReadExt;
-use futures::{Stream, StreamExt, TryStreamExt};
+use futures::Stream;
use serde_json::Value;
use switchyard_protocol::LlmClientError;
-use crate::codecs::stream::encode_response_stream_event;
-use crate::sse;
use crate::{
- AggLlmResponse, FormatId, LlmRequest, LlmResponseChunk, LlmResponseStream,
- LlmResponseStreamEvent, LlmStreamError, Result, StreamCodecRegistry, StreamTranslationState,
- TranslationEngine, TranslationPolicy, WireFormat,
+ AggLlmResponse, LlmRequest, LlmResponseStream, RawEventStream, Result, TranslationEngine,
+ TranslationPolicy, WireFormat,
};
static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =
@@ -89,12 +83,6 @@ pub fn encode_aggregated_response_with_extensions(
Ok(body)
}
-/// A stream of wire-format event objects in one format — the unframed body of an
-/// SSE response. The serving layer frames each `Value` (e.g. as an SSE
-/// `data:`/`event:` block).
-pub type RawEventStream =
- Pin<Box<dyn Stream<Item = std::result::Result<Value, LlmStreamError>> + Send>>;
-
/// Encodes a stream of IR chunks into a stream of target-format wire events.
///
/// `served_model` is exposed as the response model (via the stream state's
@@ -106,12 +94,7 @@ pub fn encode_stream(
target: WireFormat,
served_model: Option<String>,
) -> std::result::Result<RawEventStream, LlmClientError> {
- encode_stream_with_extensions(
- chunks,
- target,
- served_model,
- &switchyard_protocol::ProviderExtensions::default(),
- )
+ DEFAULT_TRANSLATION_ENGINE.encode_stream(target, chunks, served_model)
}
/// Encodes a response stream, honouring request extensions.
@@ -124,111 +107,12 @@ pub fn encode_stream_with_extensions(
served_model: Option<String>,
request_extensions: &switchyard_protocol::ProviderExtensions,
) -> std::result::Result<RawEventStream, LlmClientError> {
- let origins = crate::codex_namespaces::qualified_tool_origins(request_extensions);
- let custom_tools = crate::codex_custom_tools::custom_tool_names(request_extensions);
- let mut custom_state = crate::codex_custom_tools::CustomToolCallStreamState::default();
- let target_format: FormatId = target.into();
- // The target is always a built-in wire format, so this lookup cannot fail; a
- // failure returns as an `Err` rather than a panic.
- let codec = StreamCodecRegistry::with_builtins()
- .codec(target_format.clone())
- // Currently the only error is that the codec is missing, which is Configuration
- .map_err(|err| LlmClientError::Configuration {
- message: err.to_string(),
- })?;
-
- let served_model_for_events = served_model.clone();
- let mut state = StreamTranslationState {
- target: Some(target_format.clone()),
- target_model: served_model,
- ..Default::default()
- };
- let mut chunks = chunks;
-
- let events = try_stream! {
- while let Some(item) = chunks.next().await {
- let event = item.map_err(LlmStreamError::Client)?;
- let mut encoded =
- encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event);
- for value in &mut encoded {
- stamp_streamed_response_model(value, target, served_model_for_events.as_deref());
- crate::codex_namespaces::restore_qualified_tool_names(value, &origins);
- }
- // Argument deltas for a freeform tool cannot be expressed on the wire; the
- // rewritten completed item carries the input instead.
- let mut encoded: Vec<Value> = encoded
- .into_iter()
- .filter_map(|mut value| {
- crate::codex_custom_tools::restore_custom_tool_calls_in_event(
- &mut value,
- &custom_tools,
- &mut custom_state,
- )
- .then_some(value)
- })
- .collect();
- let terminal = encoded.pop();
- for value in encoded {
- yield value;
- }
- match (state.errored, terminal) {
- (false, Some(value)) => yield value,
- (true, Some(value)) => Err(LlmStreamError::Upstream(value))?,
- (true, None) => Err(LlmStreamError::Client(LlmClientError::ResponseTranslation(
- "stream ended after an error without a terminal event".to_string(),
- )))?,
- _ => ()
- }
- }
- for mut value in codec.finish(&mut state) {
- stamp_streamed_response_model(
- &mut value,
- target,
- served_model_for_events.as_deref(),
- );
- crate::codex_namespaces::restore_qualified_tool_names(&mut value, &origins);
- if crate::codex_custom_tools::restore_custom_tool_calls_in_event(
- &mut value,
- &custom_tools,
- &mut custom_state,
- ) {
- yield value;
- }
- }
- };
-
- Ok(Box::pin(events))
-}
-
-// The raw-response helper promises that the caller sees the model that served the
-// request. Same-format preservation bypasses provider codecs, so apply that
-// helper-specific override after replay without disturbing any other raw fields.
-fn stamp_streamed_response_model(
- event: &mut Value,
- target: WireFormat,
- served_model: Option<&str>,
-) {
- let Some(served_model) = served_model else {
- return;
- };
-
- match target {
- WireFormat::OpenAiChat => {
- if let Some(event) = event.as_object_mut() {
- event.insert("model".to_string(), Value::String(served_model.to_string()));
- }
- }
- WireFormat::OpenAiResponses => {
- if let Some(response) = event.get_mut("response").and_then(Value::as_object_mut) {
- response.insert("model".to_string(), Value::String(served_model.to_string()));
- }
- }
- WireFormat::AnthropicMessages => {
- if let Some(message) = event.get_mut("message").and_then(Value::as_object_mut) {
- message.insert("model".to_string(), Value::String(served_model.to_string()));
- }
- }
- }
+ DEFAULT_TRANSLATION_ENGINE.encode_stream_with_extensions(
+ target,
+ chunks,
+ served_model,
+ request_extensions,
+ )
}
/// Decodes provider SSE bytes into normalized stream events.
@@ -248,113 +132,7 @@ pub fn decode_stream<S>(
where
S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,
{
- let marker = sse::done_marker(source);
- let source_format: FormatId = source.into();
- // The source is always a built-in wire format, so this lookup cannot fail; a
- // failure returns as an `Err` rather than a panic.
- let codec = StreamCodecRegistry::with_builtins()
- .codec(source_format.clone())
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
- // Adapt the byte-chunk stream into an async line reader. The BufReader
- // reassembles data split across network chunks (including multi-byte UTF-8),
- // and `lines()` yields one SSE field line at a time. The stream is boxed to
- // an `io::Error` item so `into_async_read`'s error bound resolves cleanly. The
- // source error is boxed intact rather than stringified, so
- // `llm_client_error_from_io` can recover its original variant on the way out.
- let io_bytes: Pin<Box<dyn Stream<Item = std::io::Result<Vec<u8>>> + Send>> =
- Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));
- let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();
-
- let mut state = StreamTranslationState {
- source: Some(source_format.clone()),
- ..StreamTranslationState::default()
- };
- let mut frame = String::new();
- let mut saw_terminal = false;
- let mut saw_error = false;
- let stream = Box::pin(try_stream! {
- futures::pin_mut!(lines);
- while let Some(line) = lines.next().await {
- let line = line.map_err(llm_client_error_from_io)?;
- // A blank line (allowing a bare CR for CRLF streams) ends the frame.
- if line.trim_end().is_empty() {
- let parsed = sse::parse_json_sse_frame(&frame, marker)
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
- frame.clear();
- match parsed {
- sse::SseFrame::Empty => {}
- sse::SseFrame::Done => {
- saw_terminal |= source != WireFormat::AnthropicMessages;
- break;
- }
- sse::SseFrame::Data(value) => {
- saw_terminal |= sse::is_terminal_event(source, &value);
- let normalized = codec.decode_event(&mut state, &value);
- saw_error |= normalized.iter().any(|chunk| matches!(
- chunk,
- LlmResponseChunk::DecodeError { .. }
- | LlmResponseChunk::StreamError { .. }
- ));
- yield LlmResponseStreamEvent::preserved(
- source_format.clone(),
- value,
- normalized,
- );
- }
- }
- } else {
- frame.push_str(&line);
- frame.push('\n');
- }
- }
-
- // A non-standard upstream might omit the final blank line; parse a trailing
- // complete frame instead of losing its last chunk.
- #[allow(clippy::collapsible_if)]
- if !frame.trim_end().is_empty() {
- let parsed = sse::parse_json_sse_frame(&frame, marker)
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
- match parsed {
- sse::SseFrame::Done => {
- saw_terminal |= source != WireFormat::AnthropicMessages;
- }
- sse::SseFrame::Data(value) => {
- saw_terminal |= sse::is_terminal_event(source, &value);
- let normalized = codec.decode_event(&mut state, &value);
- saw_error |= normalized.iter().any(|chunk| matches!(
- chunk,
- LlmResponseChunk::DecodeError { .. }
- | LlmResponseChunk::StreamError { .. }
- ));
- yield LlmResponseStreamEvent::preserved(source_format, value, normalized);
- }
- sse::SseFrame::Empty => {}
- }
- }
-
- if !saw_terminal && !saw_error {
- Err(LlmClientError::ResponseTranslation(format!(
- "{source} stream ended before a terminal event"
- )))?;
- }
- });
- Ok(stream)
-}
-
-// Recover transport errors wrapped for `AsyncRead`; other reader failures are
-// invalid upstream responses.
-fn llm_client_error_from_io(error: std::io::Error) -> LlmClientError {
- let kind = error.kind();
- let message = error.to_string();
- match error.into_inner() {
- Some(source) => match source.downcast::<LlmClientError>() {
- Ok(error) => *error,
- Err(source) => LlmClientError::InvalidResponse { source },
- },
- None => LlmClientError::InvalidResponse {
- source: Box::new(std::io::Error::new(kind, message)),
- },
- }
+ DEFAULT_TRANSLATION_ENGINE.decode_stream(source, bytes)
}
#[cfg(test)]
@@ -368,7 +146,7 @@ mod tests {
use super::{
decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
- encode_request, encode_stream, stamp_streamed_response_model,
+ encode_request, encode_stream,
};
use crate::{LlmResponseStream, LlmStreamError, WireFormat};
@@ -744,61 +522,6 @@ mod tests {
Ok(())
}
- #[test]
- fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {
- let mut event = json!({
- "choices": [{"delta": {"content": "Hello"}}],
- "system_fingerprint": "fp_provider_specific",
- });
-
- stamp_streamed_response_model(&mut event, WireFormat::OpenAiChat, Some("served/model"));
-
- assert_eq!(event["model"], "served/model");
- assert_eq!(event["system_fingerprint"], "fp_provider_specific");
- }
-
- #[test]
- fn responses_replay_stamps_the_served_model_inside_response() {
- let mut event = json!({
- "type": "response.created",
- "response": {
- "id": "resp_1",
- "model": "provider/model",
- "provider_extension": true,
- },
- });
-
- stamp_streamed_response_model(
- &mut event,
- WireFormat::OpenAiResponses,
- Some("served/model"),
- );
-
- assert_eq!(event["response"]["model"], "served/model");
- assert_eq!(event["response"]["provider_extension"], true);
- }
-
- #[test]
- fn anthropic_replay_stamps_the_served_model_inside_message() {
- let mut event = json!({
- "type": "message_start",
- "message": {
- "id": "msg_1",
- "model": "provider/model",
- "provider_extension": true,
- },
- });
-
- stamp_streamed_response_model(
- &mut event,
- WireFormat::AnthropicMessages,
- Some("served/model"),
- );
-
- assert_eq!(event["message"]["model"], "served/model");
- assert_eq!(event["message"]["provider_extension"], true);
- }
-
#[test]
fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> {
// A multi-byte codepoint and the frame boundaries are split across
diff --git a/crates/switchyard-translation/src/sse.rs b/crates/switchyard-translation/src/sse.rs
index bf24aabbb..bd82e0ab1 100644
--- a/crates/switchyard-translation/src/sse.rs
+++ b/crates/switchyard-translation/src/sse.rs
@@ -9,8 +9,6 @@
use serde_json::Value;
-use crate::WireFormat;
-
/// Boxed, thread-safe error carried by a streamed item.
pub(crate) type BoxError = Box<dyn std::error::Error + Send + Sync>;
@@ -24,9 +22,8 @@ pub(crate) enum SseFrame {
/// The optional terminal SSE marker accepted for all supported wire formats.
/// Native Anthropic streams can end at `message_stop`, while compatible
/// providers may additionally send `[DONE]`.
-#[inline]
-pub(crate) fn done_marker(_format: WireFormat) -> Option<&'static str> {
- Some("[DONE]")
+pub(crate) const fn done_marker() -> &'static str {
+ "[DONE]"
}
/// Value of a `data` field line; the space after the colon is optional framing.
@@ -39,37 +36,7 @@ fn data_field_value(line: &str) -> Option<String> {
Some(value.strip_prefix(' ').unwrap_or(value).to_string())
}
-/// Returns whether a provider event explicitly completes its wire-format stream.
-pub(crate) fn is_terminal_event(format: WireFormat, event: &Value) -> bool {
- match format {
- WireFormat::OpenAiChat => event
- .get("choices")
- .and_then(Value::as_array)
- .into_iter()
- .flatten()
- .any(|choice| {
- choice
- .get("finish_reason")
- .and_then(Value::as_str)
- .is_some()
- }),
- WireFormat::AnthropicMessages => {
- event.get("type").and_then(Value::as_str) == Some("message_stop")
- }
- WireFormat::OpenAiResponses => matches!(
- event
- .get("type")
- .or_else(|| event.get("event"))
- .and_then(Value::as_str),
- Some("response.completed" | "response.incomplete" | "response.failed")
- ),
- }
-}
-
-pub(crate) fn parse_json_sse_frame(
- frame: &str,
- done_marker: Option<&str>,
-) -> Result<SseFrame, BoxError> {
+pub(crate) fn parse_json_sse_frame(frame: &str, done_marker: &str) -> Result<SseFrame, BoxError> {
let data = frame
.lines()
.filter(|line| !line.is_empty() && !line.starts_with(':'))
@@ -85,7 +52,7 @@ pub(crate) fn parse_json_sse_frame(
if data.is_empty() {
return Ok(SseFrame::Empty);
}
- if done_marker.is_some_and(|marker| data == marker) {
+ if data == done_marker {
return Ok(SseFrame::Done);
}
let value = serde_json::from_str::<Value>(data)?;
@@ -98,7 +65,7 @@ mod tests {
use super::*;
- const DONE: Option<&str> = Some("[DONE]");
+ const DONE: &str = done_marker();
#[test]
fn parses_a_data_line_as_json() -> Result<(), BoxError> {
@@ -170,54 +137,8 @@ mod tests {
Ok(())
}
- #[test]
- fn marker_is_only_terminal_when_configured() {
- // Without a configured marker, `[DONE]` is treated as (invalid) JSON data.
- assert!(parse_json_sse_frame("data: [DONE]\n", None).is_err());
- }
-
#[test]
fn invalid_json_is_an_error() {
assert!(parse_json_sse_frame("data: {not json}\n", DONE).is_err());
}
-
- #[test]
- fn openai_formats_terminate_on_done() {
- assert_eq!(done_marker(WireFormat::OpenAiChat), Some("[DONE]"));
- assert_eq!(done_marker(WireFormat::OpenAiResponses), Some("[DONE]"));
- }
-
- #[test]
- fn anthropic_accepts_optional_done_marker() {
- assert_eq!(done_marker(WireFormat::AnthropicMessages), Some("[DONE]"));
- }
-
- #[test]
- fn recognizes_provider_terminal_events() {
- // Each source format requires its own protocol-specific terminal event.
- assert!(is_terminal_event(
- WireFormat::OpenAiChat,
- &json!({"choices": [{"finish_reason": "stop"}]})
- ));
- assert!(is_terminal_event(
- WireFormat::AnthropicMessages,
- &json!({"type": "message_stop"})
- ));
- assert!(is_terminal_event(
- WireFormat::OpenAiResponses,
- &json!({"type": "response.completed"})
- ));
- assert!(is_terminal_event(
- WireFormat::OpenAiResponses,
- &json!({"type": "response.incomplete"})
- ));
- assert!(is_terminal_event(
- WireFormat::OpenAiResponses,
- &json!({"type": "response.failed"})
- ));
- assert!(!is_terminal_event(
- WireFormat::OpenAiChat,
- &json!({"choices": [{"finish_reason": null}]})
- ));
- }
}
diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs
index 0d78f07d2..e3a9f497d 100644
--- a/crates/switchyard-translation/tests/extension_points.rs
+++ b/crates/switchyard-translation/tests/extension_points.rs
@@ -3,8 +3,11 @@
//! Tests for custom translation and stream codec extension points.
+use futures::executor::block_on;
+use futures::{StreamExt, TryStreamExt, stream};
use pretty_assertions::assert_eq;
use serde_json::{Value, json};
+use switchyard_protocol::LlmClientError;
use switchyard_translation::codecs::{
DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
};
@@ -106,6 +109,126 @@ fn custom_stream_codec_replays_preserved_same_format_event() -> TestResult {
Ok(())
}
+#[test]
+fn custom_whole_streams_use_registered_codecs_and_outlive_the_engine() -> TestResult {
+ let mut registry = StreamCodecRegistry::new();
+ registry.register(CustomStreamCodec);
+ let engine = TranslationEngine::with_registries(FormatRegistry::new(), registry);
+ let format = FormatId::new("custom_stream");
+ let bytes = stream::iter([Ok(
+ b"data: {\"kind\":\"delta\",\"text\":\"hello\",\"vendor_only\":true}\n\ndata: {\"kind\":\"done\"}\n\n"
+ .to_vec(),
+ )]);
+ let decoded = engine.decode_stream(format.clone(), bytes)?;
+ let replayed = engine.encode_stream_with_extensions(
+ format.clone(),
+ decoded,
+ Some("replayed-model".into()),
+ &Default::default(),
+ )?;
+ let generated = engine.encode_stream(
+ format,
+ stream::iter([Ok(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "generated".into(),
+ }
+ .into())])
+ .boxed(),
+ Some("generated-model".into()),
+ )?;
+ drop(engine);
+
+ assert_eq!(
+ block_on(replayed.try_collect::<Vec<_>>())?,
+ vec![
+ json!({"kind": "delta", "text": "hello", "vendor_only": true, "backend": "replayed-model"}),
+ json!({"kind": "done", "backend": "replayed-model"}),
+ ]
+ );
+ assert_eq!(
+ block_on(generated.try_collect::<Vec<_>>())?,
+ vec![
+ json!({"kind": "delta", "text": "generated", "backend": "generated-model"}),
+ json!({"kind": "done", "backend": "generated-model"}),
+ ]
+ );
+ Ok(())
+}
+
+#[test]
+fn custom_stream_completion_uses_the_codec_rules() -> TestResult {
+ let mut registry = StreamCodecRegistry::new();
+ registry.register(CustomStreamCodec);
+ let engine = TranslationEngine::with_registries(FormatRegistry::new(), registry);
+ let cases = [
+ ("EOF after text", "", false),
+ ("EOF after an empty frame", "\n\n", false),
+ ("SSE marker alone", "data: [DONE]\n\n", false),
+ ("trailing SSE marker alone", "data: [DONE]", false),
+ ("terminal JSON event", "data: {\"kind\":\"done\"}\n\n", true),
+ (
+ "trailing terminal JSON event",
+ "data: {\"kind\":\"done\"}",
+ true,
+ ),
+ (
+ "terminal JSON event then SSE marker",
+ "data: {\"kind\":\"done\"}\n\ndata: [DONE]\n\n",
+ true,
+ ),
+ (
+ "terminal JSON event then trailing SSE marker",
+ "data: {\"kind\":\"done\"}\n\ndata: [DONE]",
+ true,
+ ),
+ ];
+ for (name, tail, is_complete) in cases {
+ let bytes =
+ format!("data: {{\"kind\":\"delta\",\"text\":\"hello\"}}\n\n{tail}").into_bytes();
+ let decoded = engine.decode_stream("custom_stream", stream::iter([Ok(bytes)]))?;
+ let mut events = block_on(decoded.collect::<Vec<_>>()).into_iter();
+ let first = events.next().expect(name)?;
+ assert_eq!(
+ first.normalized(),
+ &[LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hello".into(),
+ }],
+ "{name}"
+ );
+ let remaining = events.collect::<Result<Vec<_>, _>>();
+ if is_complete {
+ assert!(remaining.is_ok(), "{name}: {remaining:?}");
+ } else {
+ assert!(
+ matches!(
+ &remaining,
+ Err(LlmClientError::ResponseTranslation(message))
+ if message == "custom_stream stream ended before a terminal event"
+ ),
+ "{name}: {remaining:?}"
+ );
+ }
+ }
+ Ok(())
+}
+
+#[test]
+fn whole_stream_methods_report_missing_codecs_at_setup() {
+ let engine =
+ TranslationEngine::with_registries(FormatRegistry::new(), StreamCodecRegistry::new());
+ assert!(matches!(
+ engine.decode_stream("missing", stream::empty()),
+ Err(LlmClientError::ResponseTranslation(message))
+ if message == "no stream codec registered for missing"
+ ));
+ assert!(matches!(
+ engine.encode_stream("missing", stream::empty().boxed(), None),
+ Err(LlmClientError::Configuration { message })
+ if message == "no stream codec registered for missing"
+ ));
+}
+
// Verifies target capability policy can reject unsupported request features.
#[test]
fn capability_profile_can_fail_fast_when_target_cannot_accept_request_features() {
@@ -273,6 +396,18 @@ impl StreamCodec for CustomStreamCodec {
FormatId::new("custom_stream")
}
+ fn is_terminal_event(&self, event: &Value) -> bool {
+ event.get("kind").and_then(Value::as_str) == Some("done")
+ }
+
+ fn sse_done_is_terminal(&self) -> bool {
+ false
+ }
+
+ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {
+ event["backend"] = Value::String(served_model.to_string());
+ }
+
fn decode_event(
&self,
_state: &mut StreamTranslationState,
@@ -299,24 +434,31 @@ impl StreamCodec for CustomStreamCodec {
}]
})
.unwrap_or_default(),
+ Some("done") => vec![LlmResponseChunk::MessageStop { reason: None }],
_ => Vec::new(),
}
}
fn encode_event(
&self,
- _state: &mut StreamTranslationState,
+ state: &mut StreamTranslationState,
event: LlmResponseChunk,
) -> Vec<Value> {
match event {
LlmResponseChunk::TextDelta { text, .. } => {
vec![json!({"kind": "delta", "text": text})]
}
+ LlmResponseChunk::MessageStop { .. } => self.finish(state),
_ => Vec::new(),
}
}
- fn finish(&self, _state: &mut StreamTranslationState) -> Vec<Value> {
- Vec::new()
+ fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value> {
+ if state.finished {
+ Vec::new()
+ } else {
+ state.finished = true;
+ vec![json!({"kind": "done"})]
+ }
}
}