Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(translation): move stream translation into TranslationEngine (#1220)
David Gardner4f23dedaa822
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rsindex f901d875e..135fb645c 100644--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs@@ -21,6 +21,20 @@ impl StreamCodec for AnthropicMessagesStreamCodec {WireFormat::AnthropicMessages.into()}+ fn is_terminal_event(&self, event: &Value) -> bool {+ event.get("type").and_then(Value::as_str) == Some("message_stop")+ }++ fn sse_done_is_terminal(&self) -> bool {+ false+ }++ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {+ if let Some(message) = event.get_mut("message").and_then(Value::as_object_mut) {+ message.insert("model".to_string(), Value::String(served_model.to_string()));+ }+ }+fn decode_event(&self,state: &mut StreamTranslationState,@@ -745,3 +759,33 @@ fn tool_input_delta(value: &Value) -> Option<String> {_ => None,}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn recognizes_stream_completion() {+ let codec = AnthropicMessagesStreamCodec;+ assert!(!codec.sse_done_is_terminal());+ assert!(codec.is_terminal_event(&json!({"type": "message_stop"})));+ assert!(!codec.is_terminal_event(&json!({"type": "message_delta"})));+ }++ #[test]+ fn anthropic_replay_stamps_the_served_model_inside_message() {+ let mut event = json!({+ "type": "message_start",+ "message": {+ "id": "msg_1",+ "model": "provider/model",+ "provider_extension": true,+ },+ });++ AnthropicMessagesStreamCodec.stamp_streamed_response_model(&mut event, "served/model");++ assert_eq!(event["message"]["model"], "served/model");+ assert_eq!(event["message"]["provider_extension"], true);+ }+}diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rsindex 02cec4b6e..bc4da4ea2 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs@@ -22,6 +22,30 @@ impl StreamCodec for OpenAiChatStreamCodec {WireFormat::OpenAiChat.into()}+ fn is_terminal_event(&self, event: &Value) -> bool {+ event+ .get("choices")+ .and_then(Value::as_array)+ .into_iter()+ .flatten()+ .any(|choice| {+ choice+ .get("finish_reason")+ .and_then(Value::as_str)+ .is_some()+ })+ }++ fn sse_done_is_terminal(&self) -> bool {+ true+ }++ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {+ if let Some(event) = event.as_object_mut() {+ event.insert("model".to_string(), Value::String(served_model.to_string()));+ }+ }+fn decode_event(&self,state: &mut StreamTranslationState,@@ -444,3 +468,29 @@ fn openai_finish_reason(reason: Option<&str>) -> String {Some(other) => other.to_string(),}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn recognizes_stream_completion() {+ let codec = OpenAiChatStreamCodec;+ assert!(codec.sse_done_is_terminal());+ assert!(codec.is_terminal_event(&json!({"choices": [{"finish_reason": "stop"}]})));+ assert!(!codec.is_terminal_event(&json!({"choices": [{"finish_reason": null}]})));+ }++ #[test]+ fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {+ let mut event = json!({+ "choices": [{"delta": {"content": "Hello"}}],+ "system_fingerprint": "fp_provider_specific",+ });++ OpenAiChatStreamCodec.stamp_streamed_response_model(&mut event, "served/model");++ assert_eq!(event["model"], "served/model");+ assert_eq!(event["system_fingerprint"], "fp_provider_specific");+ }+}diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex b726bea71..8d4a5c3c7 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -27,6 +27,26 @@ impl StreamCodec for OpenAiResponsesStreamCodec {WireFormat::OpenAiResponses.into()}+ fn is_terminal_event(&self, event: &Value) -> bool {+ matches!(+ event+ .get("type")+ .or_else(|| event.get("event"))+ .and_then(Value::as_str),+ Some("response.completed" | "response.incomplete" | "response.failed")+ )+ }++ fn sse_done_is_terminal(&self) -> bool {+ true+ }++ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {+ if let Some(response) = event.get_mut("response").and_then(Value::as_object_mut) {+ response.insert("model".to_string(), Value::String(served_model.to_string()));+ }+ }+fn decode_event(&self,state: &mut StreamTranslationState,@@ -1226,3 +1246,40 @@ fn responses_id(state: &StreamTranslationState) -> String {.unwrap_or("resp_switchyard").to_string()}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn recognizes_stream_completion() {+ let codec = OpenAiResponsesStreamCodec;+ assert!(codec.sse_done_is_terminal());+ for kind in [+ "response.completed",+ "response.incomplete",+ "response.failed",+ ] {+ assert!(codec.is_terminal_event(&json!({"type": kind})));+ assert!(codec.is_terminal_event(&json!({"event": kind})));+ }+ assert!(!codec.is_terminal_event(&json!({"type": "response.created"})));+ }++ #[test]+ fn responses_replay_stamps_the_served_model_inside_response() {+ let mut event = json!({+ "type": "response.created",+ "response": {+ "id": "resp_1",+ "model": "provider/model",+ "provider_extension": true,+ },+ });++ OpenAiResponsesStreamCodec.stamp_streamed_response_model(&mut event, "served/model");++ assert_eq!(event["response"]["model"], "served/model");+ assert_eq!(event["response"]["provider_extension"], true);+ }+}diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rsindex 04f3b0f29..145679abf 100644--- a/crates/switchyard-translation/src/codecs/stream.rs+++ b/crates/switchyard-translation/src/codecs/stream.rs@@ -169,6 +169,20 @@ pub trait StreamCodec: Send + Sync {/// Returns the stream format handled by this codec.fn format(&self) -> FormatId;+ /// Returns whether a JSON event establishes stream completion.+ fn is_terminal_event(&self, event: &Value) -> bool;++ /// Returns whether the SSE `[DONE]` marker establishes stream completion.+ ///+ /// The marker ends consumption even when this returns `false`; a terminal JSON event+ /// must then have been observed to avoid a premature-EOF error.+ fn sse_done_is_terminal(&self) -> bool;++ /// Overrides the model reported by an encoded or preserved event.+ ///+ /// Called only when a served model was supplied, including for events emitted by `finish`.+ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str);+/// Decodes one provider event into zero or more neutral events.fn decode_event(&self,diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rsindex ae2020a87..c0865313b 100644--- a/crates/switchyard-translation/src/engine.rs+++ b/crates/switchyard-translation/src/engine.rs@@ -1,14 +1,18 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Registry-backed translation engine for buffered requests and responses.+//! Registry-backed translation engine for buffered and streaming translation.use std::collections::BTreeMap;+use std::pin::Pin;use std::sync::Arc;+use async_stream::try_stream;+use futures::io::AsyncBufReadExt;+use futures::{Stream, StreamExt, TryStreamExt};use serde_json::Value;+use switchyard_protocol::LlmClientError;-use crate::LlmResponseStreamEvent;use crate::codecs::FormatCodec;use crate::codecs::anthropic::AnthropicMessagesCodec;use crate::codecs::openai_chat::OpenAiChatCodec;@@ -21,6 +25,14 @@ use crate::error::{Result, TranslationError};use crate::format::FormatId;use crate::llm::{AggLlmResponse, LlmRequest, ProviderExtensions};use crate::policy::TranslationPolicy;+use crate::sse;+use crate::{LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, LlmStreamError};++/// A stream of wire-format event objects in one format — the unframed body of an+/// SSE response. The serving layer frames each `Value` (e.g. as an SSE+/// `data:`/`event:` block).+pub type RawEventStream =+ Pin<Box<dyn Stream<Item = std::result::Result<Value, LlmStreamError>> + Send>>;/// Encoded translation result with any diagnostics emitted along the way.#[derive(Debug)]@@ -333,6 +345,238 @@ impl TranslationEngine {let target_codec = self.stream_registry.codec(target)?;Ok(target_codec.finish(state))}++ /// Encodes a stream of IR chunks into a stream of target-format wire events.+ ///+ /// `served_model` is exposed as the response model (via the stream state's+ /// `target_model`); `None` falls back to the id observed on the source stream.+ /// The target stream codec is resolved once and reused per chunk; terminal events+ /// (`message_stop` / `response.completed`) come from `finish`.+ pub fn encode_stream(+ &self,+ target: impl Into<FormatId>,+ chunks: LlmResponseStream,+ served_model: Option<String>,+ ) -> std::result::Result<RawEventStream, LlmClientError> {+ self.encode_stream_with_extensions(+ target,+ chunks,+ served_model,+ &ProviderExtensions::default(),+ )+ }++ /// Encodes a response stream, honouring request extensions.+ ///+ /// Identical to [`Self::encode_stream`] except that Codex tool namespaces recorded on+ /// the request are restored on each encoded event.+ pub fn encode_stream_with_extensions(+ &self,+ target: impl Into<FormatId>,+ chunks: LlmResponseStream,+ served_model: Option<String>,+ request_extensions: &ProviderExtensions,+ ) -> std::result::Result<RawEventStream, LlmClientError> {+ let origins = crate::codex_namespaces::qualified_tool_origins(request_extensions);+ let custom_tools = crate::codex_custom_tools::custom_tool_names(request_extensions);+ let mut custom_state = crate::codex_custom_tools::CustomToolCallStreamState::default();+ let target_format: FormatId = target.into();+ let codec = self+ .stream_registry+ .codec(target_format.clone())+ // Currently the only error is that the codec is missing, which is Configuration+ .map_err(|err| LlmClientError::Configuration {+ message: err.to_string(),+ })?;++ let served_model_for_events = served_model.clone();+ let mut state = StreamTranslationState {+ target: Some(target_format.clone()),+ target_model: served_model,+ ..Default::default()+ };+ let mut chunks = chunks;++ let events = try_stream! {+ while let Some(item) = chunks.next().await {+ let event = item.map_err(LlmStreamError::Client)?;+ let mut encoded =+ encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event);+ for value in &mut encoded {+ if let Some(model) = served_model_for_events.as_deref() {+ codec.stamp_streamed_response_model(value, model);+ }+ crate::codex_namespaces::restore_qualified_tool_names(value, &origins);+ }+ // Argument deltas for a freeform tool cannot be expressed on the wire; the+ // rewritten completed item carries the input instead.+ let mut encoded: Vec<Value> = encoded+ .into_iter()+ .filter_map(|mut value| {+ crate::codex_custom_tools::restore_custom_tool_calls_in_event(+ &mut value,+ &custom_tools,+ &mut custom_state,+ )+ .then_some(value)+ })+ .collect();+ let terminal = encoded.pop();+ for value in encoded {+ yield value;+ }+ match (state.errored, terminal) {+ (false, Some(value)) => yield value,+ (true, Some(value)) => Err(LlmStreamError::Upstream(value))?,+ (true, None) => Err(LlmStreamError::Client(LlmClientError::ResponseTranslation(+ "stream ended after an error without a terminal event".to_string(),+ )))?,+ _ => ()+ }+ }+ for mut value in codec.finish(&mut state) {+ if let Some(model) = served_model_for_events.as_deref() {+ codec.stamp_streamed_response_model(&mut value, model);+ }+ crate::codex_namespaces::restore_qualified_tool_names(&mut value, &origins);+ if crate::codex_custom_tools::restore_custom_tool_calls_in_event(+ &mut value,+ &custom_tools,+ &mut custom_state,+ ) {+ yield value;+ }+ }+ };++ Ok(Box::pin(events))+ }++ /// Decodes provider SSE bytes into normalized stream events.+ ///+ /// Operates on raw bytes, not any HTTP client type: the caller adapts its+ /// transport's body stream into `Stream<Item = Result<Vec<u8>, _>>`. Frames are+ /// buffered across chunks (a partial frame waits for its boundary); the source+ /// stream codec is resolved once and reused for every frame.+ ///+ /// The decoder tracks the source format's protocol-specific terminal event. If+ /// EOF arrives without that required event, the stream yields a deferred+ /// [`LlmClientError::ResponseTranslation`] error after any valid decoded events.+ pub fn decode_stream<S>(+ &self,+ source: impl Into<FormatId>,+ bytes: S,+ ) -> std::result::Result<LlmResponseStream, LlmClientError>+ where+ S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,+ {+ let marker = sse::done_marker();+ let source_format: FormatId = source.into();+ let codec = self+ .stream_registry+ .codec(source_format.clone())+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ // Adapt the byte-chunk stream into an async line reader. The BufReader+ // reassembles data split across network chunks (including multi-byte UTF-8),+ // and `lines()` yields one SSE field line at a time. The stream is boxed to+ // an `io::Error` item so `into_async_read`'s error bound resolves cleanly. The+ // source error is boxed intact rather than stringified, so+ // `llm_client_error_from_io` can recover its original variant on the way out.+ let io_bytes: Pin<Box<dyn Stream<Item = std::io::Result<Vec<u8>>> + Send>> =+ Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));+ let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();++ let mut state = StreamTranslationState {+ source: Some(source_format.clone()),+ ..StreamTranslationState::default()+ };+ let mut frame = String::new();+ let mut saw_terminal = false;+ let mut saw_error = false;+ let stream = Box::pin(try_stream! {+ futures::pin_mut!(lines);+ while let Some(line) = lines.next().await {+ let line = line.map_err(llm_client_error_from_io)?;+ // A blank line (allowing a bare CR for CRLF streams) ends the frame.+ if line.trim_end().is_empty() {+ let parsed = sse::parse_json_sse_frame(&frame, marker)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ frame.clear();+ match parsed {+ sse::SseFrame::Empty => {}+ sse::SseFrame::Done => {+ saw_terminal |= codec.sse_done_is_terminal();+ break;+ }+ sse::SseFrame::Data(value) => {+ saw_terminal |= codec.is_terminal_event(&value);+ let normalized = codec.decode_event(&mut state, &value);+ saw_error |= normalized.iter().any(|chunk| matches!(+ chunk,+ LlmResponseChunk::DecodeError { .. }+ | LlmResponseChunk::StreamError { .. }+ ));+ yield LlmResponseStreamEvent::preserved(+ source_format.clone(),+ value,+ normalized,+ );+ }+ }+ } else {+ frame.push_str(&line);+ frame.push('\n');+ }+ }++ // A non-standard upstream might omit the final blank line; parse a trailing+ // complete frame instead of losing its last chunk.+ #[allow(clippy::collapsible_if)]+ if !frame.trim_end().is_empty() {+ let parsed = sse::parse_json_sse_frame(&frame, marker)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ match parsed {+ sse::SseFrame::Done => {+ saw_terminal |= codec.sse_done_is_terminal();+ }+ sse::SseFrame::Data(value) => {+ saw_terminal |= codec.is_terminal_event(&value);+ let normalized = codec.decode_event(&mut state, &value);+ saw_error |= normalized.iter().any(|chunk| matches!(+ chunk,+ LlmResponseChunk::DecodeError { .. }+ | LlmResponseChunk::StreamError { .. }+ ));+ yield LlmResponseStreamEvent::preserved(source_format.clone(), value, normalized);+ }+ sse::SseFrame::Empty => {}+ }+ }++ if !saw_terminal && !saw_error {+ Err(LlmClientError::ResponseTranslation(format!(+ "{source_format} stream ended before a terminal event"+ )))?;+ }+ });+ Ok(stream)+ }+}++// Recover transport errors wrapped for `AsyncRead`; other reader failures are+// invalid upstream responses.+fn llm_client_error_from_io(error: std::io::Error) -> LlmClientError {+ let kind = error.kind();+ let message = error.to_string();+ match error.into_inner() {+ Some(source) => match source.downcast::<LlmClientError>() {+ Ok(error) => *error,+ Err(source) => LlmClientError::InvalidResponse { source },+ },+ None => LlmClientError::InvalidResponse {+ source: Box::new(std::io::Error::new(kind, message)),+ },+ }}// Attaches source and target formats to every diagnostic emitted across both passes.diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rsindex 95ebb75e7..ef6b39104 100644--- a/crates/switchyard-translation/src/helpers.rs+++ b/crates/switchyard-translation/src/helpers.rs@@ -6,21 +6,15 @@//! streamed response — so callers can translate without threading an engine and//! policy through every call.-use std::pin::Pin;use std::sync::LazyLock;-use async_stream::try_stream;-use futures::io::AsyncBufReadExt;-use futures::{Stream, StreamExt, TryStreamExt};+use futures::Stream;use serde_json::Value;use switchyard_protocol::LlmClientError;-use crate::codecs::stream::encode_response_stream_event;-use crate::sse;use crate::{- AggLlmResponse, FormatId, LlmRequest, LlmResponseChunk, LlmResponseStream,- LlmResponseStreamEvent, LlmStreamError, Result, StreamCodecRegistry, StreamTranslationState,- TranslationEngine, TranslationPolicy, WireFormat,+ AggLlmResponse, LlmRequest, LlmResponseStream, RawEventStream, Result, TranslationEngine,+ TranslationPolicy, WireFormat,};static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =@@ -89,12 +83,6 @@ pub fn encode_aggregated_response_with_extensions(Ok(body)}-/// A stream of wire-format event objects in one format — the unframed body of an-/// SSE response. The serving layer frames each `Value` (e.g. as an SSE-/// `data:`/`event:` block).-pub type RawEventStream =- Pin<Box<dyn Stream<Item = std::result::Result<Value, LlmStreamError>> + Send>>;-/// Encodes a stream of IR chunks into a stream of target-format wire events.////// `served_model` is exposed as the response model (via the stream state's@@ -106,12 +94,7 @@ pub fn encode_stream(target: WireFormat,served_model: Option<String>,) -> std::result::Result<RawEventStream, LlmClientError> {- encode_stream_with_extensions(- chunks,- target,- served_model,- &switchyard_protocol::ProviderExtensions::default(),- )+ DEFAULT_TRANSLATION_ENGINE.encode_stream(target, chunks, served_model)}/// Encodes a response stream, honouring request extensions.@@ -124,111 +107,12 @@ pub fn encode_stream_with_extensions(served_model: Option<String>,request_extensions: &switchyard_protocol::ProviderExtensions,) -> std::result::Result<RawEventStream, LlmClientError> {- let origins = crate::codex_namespaces::qualified_tool_origins(request_extensions);- let custom_tools = crate::codex_custom_tools::custom_tool_names(request_extensions);- let mut custom_state = crate::codex_custom_tools::CustomToolCallStreamState::default();- let target_format: FormatId = target.into();- // The target is always a built-in wire format, so this lookup cannot fail; a- // failure returns as an `Err` rather than a panic.- let codec = StreamCodecRegistry::with_builtins()- .codec(target_format.clone())- // Currently the only error is that the codec is missing, which is Configuration- .map_err(|err| LlmClientError::Configuration {- message: err.to_string(),- })?;-- let served_model_for_events = served_model.clone();- let mut state = StreamTranslationState {- target: Some(target_format.clone()),- target_model: served_model,- ..Default::default()- };- let mut chunks = chunks;-- let events = try_stream! {- while let Some(item) = chunks.next().await {- let event = item.map_err(LlmStreamError::Client)?;- let mut encoded =- encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event);- for value in &mut encoded {- stamp_streamed_response_model(value, target, served_model_for_events.as_deref());- crate::codex_namespaces::restore_qualified_tool_names(value, &origins);- }- // Argument deltas for a freeform tool cannot be expressed on the wire; the- // rewritten completed item carries the input instead.- let mut encoded: Vec<Value> = encoded- .into_iter()- .filter_map(|mut value| {- crate::codex_custom_tools::restore_custom_tool_calls_in_event(- &mut value,- &custom_tools,- &mut custom_state,- )- .then_some(value)- })- .collect();- let terminal = encoded.pop();- for value in encoded {- yield value;- }- match (state.errored, terminal) {- (false, Some(value)) => yield value,- (true, Some(value)) => Err(LlmStreamError::Upstream(value))?,- (true, None) => Err(LlmStreamError::Client(LlmClientError::ResponseTranslation(- "stream ended after an error without a terminal event".to_string(),- )))?,- _ => ()- }- }- for mut value in codec.finish(&mut state) {- stamp_streamed_response_model(- &mut value,- target,- served_model_for_events.as_deref(),- );- crate::codex_namespaces::restore_qualified_tool_names(&mut value, &origins);- if crate::codex_custom_tools::restore_custom_tool_calls_in_event(- &mut value,- &custom_tools,- &mut custom_state,- ) {- yield value;- }- }- };-- Ok(Box::pin(events))-}--// The raw-response helper promises that the caller sees the model that served the-// request. Same-format preservation bypasses provider codecs, so apply that-// helper-specific override after replay without disturbing any other raw fields.-fn stamp_streamed_response_model(- event: &mut Value,- target: WireFormat,- served_model: Option<&str>,-) {- let Some(served_model) = served_model else {- return;- };-- match target {- WireFormat::OpenAiChat => {- if let Some(event) = event.as_object_mut() {- event.insert("model".to_string(), Value::String(served_model.to_string()));- }- }- WireFormat::OpenAiResponses => {- if let Some(response) = event.get_mut("response").and_then(Value::as_object_mut) {- response.insert("model".to_string(), Value::String(served_model.to_string()));- }- }- WireFormat::AnthropicMessages => {- if let Some(message) = event.get_mut("message").and_then(Value::as_object_mut) {- message.insert("model".to_string(), Value::String(served_model.to_string()));- }- }- }+ DEFAULT_TRANSLATION_ENGINE.encode_stream_with_extensions(+ target,+ chunks,+ served_model,+ request_extensions,+ )}/// Decodes provider SSE bytes into normalized stream events.@@ -248,113 +132,7 @@ pub fn decode_stream<S>(whereS: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,{- let marker = sse::done_marker(source);- let source_format: FormatId = source.into();- // The source is always a built-in wire format, so this lookup cannot fail; a- // failure returns as an `Err` rather than a panic.- let codec = StreamCodecRegistry::with_builtins()- .codec(source_format.clone())- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;- // Adapt the byte-chunk stream into an async line reader. The BufReader- // reassembles data split across network chunks (including multi-byte UTF-8),- // and `lines()` yields one SSE field line at a time. The stream is boxed to- // an `io::Error` item so `into_async_read`'s error bound resolves cleanly. The- // source error is boxed intact rather than stringified, so- // `llm_client_error_from_io` can recover its original variant on the way out.- let io_bytes: Pin<Box<dyn Stream<Item = std::io::Result<Vec<u8>>> + Send>> =- Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));- let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();-- let mut state = StreamTranslationState {- source: Some(source_format.clone()),- ..StreamTranslationState::default()- };- let mut frame = String::new();- let mut saw_terminal = false;- let mut saw_error = false;- let stream = Box::pin(try_stream! {- futures::pin_mut!(lines);- while let Some(line) = lines.next().await {- let line = line.map_err(llm_client_error_from_io)?;- // A blank line (allowing a bare CR for CRLF streams) ends the frame.- if line.trim_end().is_empty() {- let parsed = sse::parse_json_sse_frame(&frame, marker)- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;- frame.clear();- match parsed {- sse::SseFrame::Empty => {}- sse::SseFrame::Done => {- saw_terminal |= source != WireFormat::AnthropicMessages;- break;- }- sse::SseFrame::Data(value) => {- saw_terminal |= sse::is_terminal_event(source, &value);- let normalized = codec.decode_event(&mut state, &value);- saw_error |= normalized.iter().any(|chunk| matches!(- chunk,- LlmResponseChunk::DecodeError { .. }- | LlmResponseChunk::StreamError { .. }- ));- yield LlmResponseStreamEvent::preserved(- source_format.clone(),- value,- normalized,- );- }- }- } else {- frame.push_str(&line);- frame.push('\n');- }- }-- // A non-standard upstream might omit the final blank line; parse a trailing- // complete frame instead of losing its last chunk.- #[allow(clippy::collapsible_if)]- if !frame.trim_end().is_empty() {- let parsed = sse::parse_json_sse_frame(&frame, marker)- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;- match parsed {- sse::SseFrame::Done => {- saw_terminal |= source != WireFormat::AnthropicMessages;- }- sse::SseFrame::Data(value) => {- saw_terminal |= sse::is_terminal_event(source, &value);- let normalized = codec.decode_event(&mut state, &value);- saw_error |= normalized.iter().any(|chunk| matches!(- chunk,- LlmResponseChunk::DecodeError { .. }- | LlmResponseChunk::StreamError { .. }- ));- yield LlmResponseStreamEvent::preserved(source_format, value, normalized);- }- sse::SseFrame::Empty => {}- }- }-- if !saw_terminal && !saw_error {- Err(LlmClientError::ResponseTranslation(format!(- "{source} stream ended before a terminal event"- )))?;- }- });- Ok(stream)-}--// Recover transport errors wrapped for `AsyncRead`; other reader failures are-// invalid upstream responses.-fn llm_client_error_from_io(error: std::io::Error) -> LlmClientError {- let kind = error.kind();- let message = error.to_string();- match error.into_inner() {- Some(source) => match source.downcast::<LlmClientError>() {- Ok(error) => *error,- Err(source) => LlmClientError::InvalidResponse { source },- },- None => LlmClientError::InvalidResponse {- source: Box::new(std::io::Error::new(kind, message)),- },- }+ DEFAULT_TRANSLATION_ENGINE.decode_stream(source, bytes)}#[cfg(test)]@@ -368,7 +146,7 @@ mod tests {use super::{decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,- encode_request, encode_stream, stamp_streamed_response_model,+ encode_request, encode_stream,};use crate::{LlmResponseStream, LlmStreamError, WireFormat};@@ -744,61 +522,6 @@ mod tests {Ok(())}- #[test]- fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {- let mut event = json!({- "choices": [{"delta": {"content": "Hello"}}],- "system_fingerprint": "fp_provider_specific",- });-- stamp_streamed_response_model(&mut event, WireFormat::OpenAiChat, Some("served/model"));-- assert_eq!(event["model"], "served/model");- assert_eq!(event["system_fingerprint"], "fp_provider_specific");- }-- #[test]- fn responses_replay_stamps_the_served_model_inside_response() {- let mut event = json!({- "type": "response.created",- "response": {- "id": "resp_1",- "model": "provider/model",- "provider_extension": true,- },- });-- stamp_streamed_response_model(- &mut event,- WireFormat::OpenAiResponses,- Some("served/model"),- );-- assert_eq!(event["response"]["model"], "served/model");- assert_eq!(event["response"]["provider_extension"], true);- }-- #[test]- fn anthropic_replay_stamps_the_served_model_inside_message() {- let mut event = json!({- "type": "message_start",- "message": {- "id": "msg_1",- "model": "provider/model",- "provider_extension": true,- },- });-- stamp_streamed_response_model(- &mut event,- WireFormat::AnthropicMessages,- Some("served/model"),- );-- assert_eq!(event["message"]["model"], "served/model");- assert_eq!(event["message"]["provider_extension"], true);- }-#[test]fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> {// A multi-byte codepoint and the frame boundaries are split acrossdiff --git a/crates/switchyard-translation/src/sse.rs b/crates/switchyard-translation/src/sse.rsindex bf24aabbb..bd82e0ab1 100644--- a/crates/switchyard-translation/src/sse.rs+++ b/crates/switchyard-translation/src/sse.rs@@ -9,8 +9,6 @@use serde_json::Value;-use crate::WireFormat;-/// Boxed, thread-safe error carried by a streamed item.pub(crate) type BoxError = Box<dyn std::error::Error + Send + Sync>;@@ -24,9 +22,8 @@ pub(crate) enum SseFrame {/// The optional terminal SSE marker accepted for all supported wire formats./// Native Anthropic streams can end at `message_stop`, while compatible/// providers may additionally send `[DONE]`.-#[inline]-pub(crate) fn done_marker(_format: WireFormat) -> Option<&'static str> {- Some("[DONE]")+pub(crate) const fn done_marker() -> &'static str {+ "[DONE]"}/// Value of a `data` field line; the space after the colon is optional framing.@@ -39,37 +36,7 @@ fn data_field_value(line: &str) -> Option<String> {Some(value.strip_prefix(' ').unwrap_or(value).to_string())}-/// Returns whether a provider event explicitly completes its wire-format stream.-pub(crate) fn is_terminal_event(format: WireFormat, event: &Value) -> bool {- match format {- WireFormat::OpenAiChat => event- .get("choices")- .and_then(Value::as_array)- .into_iter()- .flatten()- .any(|choice| {- choice- .get("finish_reason")- .and_then(Value::as_str)- .is_some()- }),- WireFormat::AnthropicMessages => {- event.get("type").and_then(Value::as_str) == Some("message_stop")- }- WireFormat::OpenAiResponses => matches!(- event- .get("type")- .or_else(|| event.get("event"))- .and_then(Value::as_str),- Some("response.completed" | "response.incomplete" | "response.failed")- ),- }-}--pub(crate) fn parse_json_sse_frame(- frame: &str,- done_marker: Option<&str>,-) -> Result<SseFrame, BoxError> {+pub(crate) fn parse_json_sse_frame(frame: &str, done_marker: &str) -> Result<SseFrame, BoxError> {let data = frame.lines().filter(|line| !line.is_empty() && !line.starts_with(':'))@@ -85,7 +52,7 @@ pub(crate) fn parse_json_sse_frame(if data.is_empty() {return Ok(SseFrame::Empty);}- if done_marker.is_some_and(|marker| data == marker) {+ if data == done_marker {return Ok(SseFrame::Done);}let value = serde_json::from_str::<Value>(data)?;@@ -98,7 +65,7 @@ mod tests {use super::*;- const DONE: Option<&str> = Some("[DONE]");+ const DONE: &str = done_marker();#[test]fn parses_a_data_line_as_json() -> Result<(), BoxError> {@@ -170,54 +137,8 @@ mod tests {Ok(())}- #[test]- fn marker_is_only_terminal_when_configured() {- // Without a configured marker, `[DONE]` is treated as (invalid) JSON data.- assert!(parse_json_sse_frame("data: [DONE]\n", None).is_err());- }-#[test]fn invalid_json_is_an_error() {assert!(parse_json_sse_frame("data: {not json}\n", DONE).is_err());}-- #[test]- fn openai_formats_terminate_on_done() {- assert_eq!(done_marker(WireFormat::OpenAiChat), Some("[DONE]"));- assert_eq!(done_marker(WireFormat::OpenAiResponses), Some("[DONE]"));- }-- #[test]- fn anthropic_accepts_optional_done_marker() {- assert_eq!(done_marker(WireFormat::AnthropicMessages), Some("[DONE]"));- }-- #[test]- fn recognizes_provider_terminal_events() {- // Each source format requires its own protocol-specific terminal event.- assert!(is_terminal_event(- WireFormat::OpenAiChat,- &json!({"choices": [{"finish_reason": "stop"}]})- ));- assert!(is_terminal_event(- WireFormat::AnthropicMessages,- &json!({"type": "message_stop"})- ));- assert!(is_terminal_event(- WireFormat::OpenAiResponses,- &json!({"type": "response.completed"})- ));- assert!(is_terminal_event(- WireFormat::OpenAiResponses,- &json!({"type": "response.incomplete"})- ));- assert!(is_terminal_event(- WireFormat::OpenAiResponses,- &json!({"type": "response.failed"})- ));- assert!(!is_terminal_event(- WireFormat::OpenAiChat,- &json!({"choices": [{"finish_reason": null}]})- ));- }}diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rsindex 0d78f07d2..e3a9f497d 100644--- a/crates/switchyard-translation/tests/extension_points.rs+++ b/crates/switchyard-translation/tests/extension_points.rs@@ -3,8 +3,11 @@//! Tests for custom translation and stream codec extension points.+use futures::executor::block_on;+use futures::{StreamExt, TryStreamExt, stream};use pretty_assertions::assert_eq;use serde_json::{Value, json};+use switchyard_protocol::LlmClientError;use switchyard_translation::codecs::{DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,};@@ -106,6 +109,126 @@ fn custom_stream_codec_replays_preserved_same_format_event() -> TestResult {Ok(())}+#[test]+fn custom_whole_streams_use_registered_codecs_and_outlive_the_engine() -> TestResult {+ let mut registry = StreamCodecRegistry::new();+ registry.register(CustomStreamCodec);+ let engine = TranslationEngine::with_registries(FormatRegistry::new(), registry);+ let format = FormatId::new("custom_stream");+ let bytes = stream::iter([Ok(+ b"data: {\"kind\":\"delta\",\"text\":\"hello\",\"vendor_only\":true}\n\ndata: {\"kind\":\"done\"}\n\n"+ .to_vec(),+ )]);+ let decoded = engine.decode_stream(format.clone(), bytes)?;+ let replayed = engine.encode_stream_with_extensions(+ format.clone(),+ decoded,+ Some("replayed-model".into()),+ &Default::default(),+ )?;+ let generated = engine.encode_stream(+ format,+ stream::iter([Ok(LlmResponseChunk::TextDelta {+ index: 0,+ text: "generated".into(),+ }+ .into())])+ .boxed(),+ Some("generated-model".into()),+ )?;+ drop(engine);++ assert_eq!(+ block_on(replayed.try_collect::<Vec<_>>())?,+ vec![+ json!({"kind": "delta", "text": "hello", "vendor_only": true, "backend": "replayed-model"}),+ json!({"kind": "done", "backend": "replayed-model"}),+ ]+ );+ assert_eq!(+ block_on(generated.try_collect::<Vec<_>>())?,+ vec![+ json!({"kind": "delta", "text": "generated", "backend": "generated-model"}),+ json!({"kind": "done", "backend": "generated-model"}),+ ]+ );+ Ok(())+}++#[test]+fn custom_stream_completion_uses_the_codec_rules() -> TestResult {+ let mut registry = StreamCodecRegistry::new();+ registry.register(CustomStreamCodec);+ let engine = TranslationEngine::with_registries(FormatRegistry::new(), registry);+ let cases = [+ ("EOF after text", "", false),+ ("EOF after an empty frame", "\n\n", false),+ ("SSE marker alone", "data: [DONE]\n\n", false),+ ("trailing SSE marker alone", "data: [DONE]", false),+ ("terminal JSON event", "data: {\"kind\":\"done\"}\n\n", true),+ (+ "trailing terminal JSON event",+ "data: {\"kind\":\"done\"}",+ true,+ ),+ (+ "terminal JSON event then SSE marker",+ "data: {\"kind\":\"done\"}\n\ndata: [DONE]\n\n",+ true,+ ),+ (+ "terminal JSON event then trailing SSE marker",+ "data: {\"kind\":\"done\"}\n\ndata: [DONE]",+ true,+ ),+ ];+ for (name, tail, is_complete) in cases {+ let bytes =+ format!("data: {{\"kind\":\"delta\",\"text\":\"hello\"}}\n\n{tail}").into_bytes();+ let decoded = engine.decode_stream("custom_stream", stream::iter([Ok(bytes)]))?;+ let mut events = block_on(decoded.collect::<Vec<_>>()).into_iter();+ let first = events.next().expect(name)?;+ assert_eq!(+ first.normalized(),+ &[LlmResponseChunk::TextDelta {+ index: 0,+ text: "hello".into(),+ }],+ "{name}"+ );+ let remaining = events.collect::<Result<Vec<_>, _>>();+ if is_complete {+ assert!(remaining.is_ok(), "{name}: {remaining:?}");+ } else {+ assert!(+ matches!(+ &remaining,+ Err(LlmClientError::ResponseTranslation(message))+ if message == "custom_stream stream ended before a terminal event"+ ),+ "{name}: {remaining:?}"+ );+ }+ }+ Ok(())+}++#[test]+fn whole_stream_methods_report_missing_codecs_at_setup() {+ let engine =+ TranslationEngine::with_registries(FormatRegistry::new(), StreamCodecRegistry::new());+ assert!(matches!(+ engine.decode_stream("missing", stream::empty()),+ Err(LlmClientError::ResponseTranslation(message))+ if message == "no stream codec registered for missing"+ ));+ assert!(matches!(+ engine.encode_stream("missing", stream::empty().boxed(), None),+ Err(LlmClientError::Configuration { message })+ if message == "no stream codec registered for missing"+ ));+}+// Verifies target capability policy can reject unsupported request features.#[test]fn capability_profile_can_fail_fast_when_target_cannot_accept_request_features() {@@ -273,6 +396,18 @@ impl StreamCodec for CustomStreamCodec {FormatId::new("custom_stream")}+ fn is_terminal_event(&self, event: &Value) -> bool {+ event.get("kind").and_then(Value::as_str) == Some("done")+ }++ fn sse_done_is_terminal(&self) -> bool {+ false+ }++ fn stamp_streamed_response_model(&self, event: &mut Value, served_model: &str) {+ event["backend"] = Value::String(served_model.to_string());+ }+fn decode_event(&self,_state: &mut StreamTranslationState,@@ -299,24 +434,31 @@ impl StreamCodec for CustomStreamCodec {}]}).unwrap_or_default(),+ Some("done") => vec![LlmResponseChunk::MessageStop { reason: None }],_ => Vec::new(),}}fn encode_event(&self,- _state: &mut StreamTranslationState,+ state: &mut StreamTranslationState,event: LlmResponseChunk,) -> Vec<Value> {match event {LlmResponseChunk::TextDelta { text, .. } => {vec![json!({"kind": "delta", "text": text})]}+ LlmResponseChunk::MessageStop { .. } => self.finish(state),_ => Vec::new(),}}- fn finish(&self, _state: &mut StreamTranslationState) -> Vec<Value> {- Vec::new()+ fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value> {+ if state.finished {+ Vec::new()+ } else {+ state.finished = true;+ vec![json!({"kind": "done"})]+ }}}