Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(translation): preserve raw stream events (#192)

Bryan Bednarski091bc89c8425
diff --git a/crates/libsy/examples/streaming_agent.rs b/crates/libsy/examples/streaming_agent.rs
index 1b7e877c5..47a3918aa 100644
--- a/crates/libsy/examples/streaming_agent.rs
+++ b/crates/libsy/examples/streaming_agent.rs
@@ -7,7 +7,8 @@
//! offloaded as a [`Step::CallLlm`]. The agent serves it with a *streaming* response
//! ([`LlmResponse::Stream`]) rather than a buffered one. That stream rides untouched
//! through the algorithm and returns as [`Step::ReturnToAgent`], where the agent drives it
-//! and prints each [`LlmResponseChunk`] as it arrives — true token streaming end to end.
+//! and prints the normalized [`LlmResponseChunk`]s in each event as they arrive — true
+//! token streaming end to end.
//! Contrast [`Algorithm::run`], which would aggregate the same stream into one buffered
//! answer. Run with:
//! cargo run -p libsy --example streaming_agent
@@ -40,7 +41,8 @@ fn streaming_response(model: &str, tokens: &[&str]) -> Response {
reason: Some("stop".to_string()),
});
- let stream: LlmResponseStream = futures::stream::iter(chunks.into_iter().map(Ok)).boxed();
+ let stream: LlmResponseStream =
+ futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed();
Response {
llm_response: LlmResponse::Stream(stream),
metadata: None,
@@ -81,15 +83,17 @@ async fn main() -> Result<()> {
}
Step::ReturnToAgent(response) => match response.llm_response {
// The stream reached the agent untouched: print each token as it arrives.
- LlmResponse::Stream(mut chunks) => {
+ LlmResponse::Stream(mut events) => {
print!("agent sees: ");
- while let Some(chunk) = chunks.next().await {
- let chunk = chunk.map_err(|error| {
+ while let Some(event) = events.next().await {
+ let event = event.map_err(|error| {
LibsyError::external("reading response stream", error)
})?;
- if let LlmResponseChunk::TextDelta { text, .. } = chunk {
- print!("{text}");
- std::io::stdout().flush().ok();
+ for chunk in event.normalized() {
+ if let LlmResponseChunk::TextDelta { text, .. } = chunk {
+ print!("{text}");
+ std::io::stdout().flush().ok();
+ }
}
}
println!();
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index c5d81e4bf..4b8fc4c5e 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -330,7 +330,7 @@ mod tests {
fn streamed(chunks: Vec<LlmResponseChunk>) -> Response {
Response {
llm_response: LlmResponse::Stream(
- futures::stream::iter(chunks.into_iter().map(Ok)).boxed(),
+ futures::stream::iter(chunks.into_iter().map(|chunk| Ok(chunk.into()))).boxed(),
),
metadata: None,
}
@@ -338,7 +338,7 @@ mod tests {
fn streamed_then_failing(chunk: LlmResponseChunk) -> Response {
let items = futures::stream::iter([
- Ok(chunk),
+ Ok(chunk.into()),
Err(LlmClientError::Timeout {
source: Box::new(std::io::Error::other("stream died")),
}),
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 69ac9d1b8..bd6212cae 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -17,10 +17,13 @@ use futures::{Stream, StreamExt};
use parking_lot::Mutex;
use tracing::Instrument;
-/// The request/response protocol types, re-exported from [`switchyard_protocol`].
-/// [`LlmRequest`] is the normalized request; [`AggLlmResponse`] is the buffered response;
-/// [`LlmResponseChunk`] is one streaming event; [`LlmResponse`] is the streamed response
-/// (a live [`LlmResponseStream`] or the terminal aggregate).
+/// The request/response protocol types come from [`switchyard_protocol`].
+/// [`switchyard_protocol::LlmRequest`] is the normalized request;
+/// [`switchyard_protocol::AggLlmResponse`] is the buffered response;
+/// [`switchyard_protocol::LlmResponseChunk`] is normalized streaming content;
+/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and
+/// [`switchyard_protocol::LlmResponse`] carries either a live
+/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
use switchyard_protocol::{
Context, Decision, LlmClientError, Request, Response, RoutedLlmClient, Signals, Usage,
};
@@ -971,7 +974,13 @@ mod tests {
_request: Request,
_decision: Arc<dyn Decision>,
) -> std::result::Result<Response, LlmClientError> {
- let stream = futures::stream::iter(self.chunks.clone().into_iter().map(Ok)).boxed();
+ let stream = futures::stream::iter(
+ self.chunks
+ .clone()
+ .into_iter()
+ .map(|chunk| Ok(chunk.into())),
+ )
+ .boxed();
Ok(Response {
llm_response: LlmResponse::Stream(stream),
metadata: None,
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index f972bd0c1..ceeed9240 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -48,7 +48,7 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{Driver, LibsyError, Result};
use switchyard_protocol::{
AggLlmResponse, Context, Decision, LlmClientError, LlmRequest, LlmResponse, LlmResponseChunk,
- LlmResponseStream, Request, Response, Usage,
+ LlmResponseStream, LlmResponseStreamEvent, Request, Response, Usage,
};
const METRICS_SCOPE: &str = "switchyard";
@@ -364,7 +364,7 @@ struct ObservedClientStream {
}
impl Stream for ObservedClientStream {
- type Item = std::result::Result<LlmResponseChunk, LlmClientError>;
+ type Item = std::result::Result<LlmResponseStreamEvent, LlmClientError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
match self.stream.as_mut().poll_next(cx) {
@@ -395,32 +395,48 @@ struct ClientStreamObserver {
}
impl ClientStreamObserver {
- fn observe(&mut self, item: &std::result::Result<LlmResponseChunk, LlmClientError>) -> bool {
+ fn observe(
+ &mut self,
+ item: &std::result::Result<LlmResponseStreamEvent, LlmClientError>,
+ ) -> bool {
match item {
- Ok(LlmResponseChunk::MessageStart { id, model }) => {
+ Ok(event) => {
+ for chunk in event.normalized() {
+ self.observe_chunk(chunk);
+ if self.terminal {
+ break;
+ }
+ }
+ }
+ Err(error) => {
+ let error_type = llm_client_error_type(error);
+ record_client_error(&self.span, &error_type, error);
+ self.terminal = true;
+ }
+ }
+ self.terminal
+ }
+
+ fn observe_chunk(&mut self, chunk: &LlmResponseChunk) {
+ match chunk {
+ LlmResponseChunk::MessageStart { id, model } => {
record_optional(&self.span, "gen_ai.response.id", id.as_deref());
record_optional(&self.span, "gen_ai.response.model", model.as_deref());
}
- Ok(LlmResponseChunk::Usage(usage)) => record_gen_ai_usage(&self.span, usage),
- Ok(LlmResponseChunk::MessageStop { reason }) => {
+ LlmResponseChunk::Usage(usage) => record_gen_ai_usage(&self.span, usage),
+ LlmResponseChunk::MessageStop { reason } => {
record_finish_reasons(&self.span, reason.iter().cloned());
}
- Ok(LlmResponseChunk::DecodeError { message }) => {
+ LlmResponseChunk::DecodeError { message } => {
record_client_error(&self.span, "response_translation", message);
self.terminal = true;
}
- Ok(LlmResponseChunk::StreamError { message }) => {
+ LlmResponseChunk::StreamError { message } => {
record_client_error(&self.span, "502", message);
self.terminal = true;
}
- Err(error) => {
- let error_type = llm_client_error_type(error);
- record_client_error(&self.span, &error_type, error);
- self.terminal = true;
- }
_ => {}
}
- self.terminal
}
fn complete(&mut self) {
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index f03fee898..ca361ff63 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -40,7 +40,8 @@ use switchyard_protocol::{
Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,
};
use switchyard_protocol::{
- LlmClientError, LlmResponseChunk, StopReason, text_request, text_response,
+ LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request,
+ text_response,
};
#[derive(Debug, thiserror::Error)]
@@ -758,16 +759,16 @@ impl RoutedLlmClient for StreamingUsageClient {
cache: Usage::cache_details(Some(8), None),
..Usage::default()
};
- let chunks = vec![
- Ok(LlmResponseChunk::MessageStart {
+ let chunks = vec![Ok(LlmResponseStreamEvent::new(vec![
+ LlmResponseChunk::MessageStart {
id: Some("obs-stream-response".to_string()),
model: Some(decision.selected_model().to_string()),
- }),
- Ok(LlmResponseChunk::Usage(usage)),
- Ok(LlmResponseChunk::MessageStop {
+ },
+ LlmResponseChunk::Usage(usage),
+ LlmResponseChunk::MessageStop {
reason: Some("end_turn".to_string()),
- }),
- ];
+ },
+ ]))];
Ok(Response {
llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))),
metadata: None,
diff --git a/crates/protocol/README.md b/crates/protocol/README.md
index 555debc33..56adc1701 100644
--- a/crates/protocol/README.md
+++ b/crates/protocol/README.md
@@ -19,7 +19,7 @@ serde_json = "1"
| Conversation | [`LlmRequest`], [`Message`], [`InstructionBlock`], [`ContentBlock`] |
| Tools | [`ToolDefinition`], [`ToolChoice`], [`ToolCall`], [`ToolResult`] |
| Response | [`AggLlmResponse`], [`ResponseOutput`], [`Usage`], [`StopReason`] |
-| Streaming | [`LlmResponse`], [`LlmResponseChunk`], [`LlmResponseStream`] |
+| Streaming | [`LlmResponse`], [`LlmResponseStream`], [`LlmResponseStreamEvent`], [`LlmResponseChunk`], [`ProviderStreamEvent`] |
| Envelope | [`Context`], [`Request`], [`Response`], [`Metadata`] |
| Routing I/O | [`Decision`], [`RoutedLlmClient`], [`LlmClientError`] |
| Wire identity | [`WireFormat`], [`FormatId`] |
@@ -102,7 +102,9 @@ assert_eq!(request.llm_request.tools[0].name, "lookup_metric");
## Response forms
[`LlmResponse`] contains either a completed [`AggLlmResponse`] or a
-single-consumption [`LlmResponseStream`] of [`LlmResponseChunk`] values. See
+single-consumption [`LlmResponseStream`] of [`LlmResponseStreamEvent`] values.
+Each event carries provider-neutral [`LlmResponseChunk`] values and may retain
+one opaque [`ProviderStreamEvent`] for same-format parsed-JSON-value replay. See
[`LlmResponse::into_agg`] for aggregation and [`PreservationMetadata`],
[`Usage`], and [`Metadata`] for the data retained around a response.
diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs
index ce7b15d92..e51b68dce 100644
--- a/crates/protocol/src/stream.rs
+++ b/crates/protocol/src/stream.rs
@@ -1,9 +1,9 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Streaming half of the neutral IR: incremental response chunks ([`LlmResponseChunk`])
-//! and the streamed response ([`LlmResponse`]) that carries either a live stream of them
-//! or the terminal [`AggLlmResponse`].
+//! Streaming half of the neutral IR: incremental response chunks ([`LlmResponseChunk`]),
+//! their stream envelope ([`LlmResponseStreamEvent`]), and the streamed response
+//! ([`LlmResponse`]) that carries either a live stream or the terminal [`AggLlmResponse`].
use std::collections::BTreeMap;
use std::pin::Pin;
@@ -14,6 +14,7 @@ use serde_json::Value;
use crate::{
LlmClientError,
+ format::FormatId,
llm::{AggLlmResponse, ContentBlock, ResponseOutput, Role, StopReason, ToolCall, Usage},
};
@@ -22,12 +23,96 @@ use crate::{
/// code to propagate; 502 matches how a failed upstream call surfaces elsewhere.
const MID_STREAM_UPSTREAM_STATUS: u16 = 502;
-/// A boxed, `Send` stream of [`LlmResponseChunk`]s — the token-by-token output of a
-/// streaming backend. Each item may fail independently mid-stream.
+/// A boxed, `Send` stream of response events. Each item may fail independently mid-stream.
pub type LlmResponseStream =
- Pin<Box<dyn Stream<Item = Result<LlmResponseChunk, LlmClientError>> + Send>>;
+ Pin<Box<dyn Stream<Item = Result<LlmResponseStreamEvent, LlmClientError>> + Send>>;
-/// A model response: either a live [`Stream`](LlmResponse::Stream) of chunks or a
+/// Parsed provider event retained for same-format replay.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct ProviderStreamEvent {
+ source: FormatId,
+ raw: Value,
+}
+
+impl ProviderStreamEvent {
+ /// Source wire format of the retained event.
+ pub fn source(&self) -> &FormatId {
+ &self.source
+ }
+
+ /// Parsed provider JSON retained for replay.
+ pub fn raw(&self) -> &Value {
+ &self.raw
+ }
+
+ /// Consumes the preservation value into its source format and parsed JSON.
+ pub fn into_parts(self) -> (FormatId, Value) {
+ (self.source, self.raw)
+ }
+}
+
+/// One streaming item crossing the host/algorithm boundary.
+///
+/// `normalized` contains only provider-neutral chunks. `preservation` is opaque to
+/// algorithms and is interpreted by `switchyard-translation` for same-format replay.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct LlmResponseStreamEvent {
+ preservation: Option<ProviderStreamEvent>,
+ normalized: Vec<LlmResponseChunk>,
+}
+
+impl LlmResponseStreamEvent {
+ /// Creates an event containing only normalized chunks.
+ pub fn new(normalized: Vec<LlmResponseChunk>) -> Self {
+ Self {
+ preservation: None,
+ normalized,
+ }
+ }
+
+ /// Creates an event with parsed provider JSON retained for replay.
+ pub fn preserved(
+ source: impl Into<FormatId>,
+ raw: Value,
+ normalized: Vec<LlmResponseChunk>,
+ ) -> Self {
+ Self {
+ preservation: Some(ProviderStreamEvent {
+ source: source.into(),
+ raw,
+ }),
+ normalized,
+ }
+ }
+
+ /// Retained provider event, when this event came directly from a provider.
+ pub fn preservation(&self) -> Option<&ProviderStreamEvent> {
+ self.preservation.as_ref()
+ }
+
+ /// Provider-neutral chunks carried by this event.
+ pub fn normalized(&self) -> &[LlmResponseChunk] {
+ &self.normalized
+ }
+
+ /// Consumes the event into its preservation and normalized content.
+ pub fn into_parts(self) -> (Option<ProviderStreamEvent>, Vec<LlmResponseChunk>) {
+ (self.preservation, self.normalized)
+ }
+
+ /// Replaces semantic content and drops raw replay data that no longer describes it.
+ pub fn replace_normalized(self, normalized: Vec<LlmResponseChunk>) -> Self {
+ Self::new(normalized)
+ }
+}
+
+impl From<LlmResponseChunk> for LlmResponseStreamEvent {
+ fn from(chunk: LlmResponseChunk) -> Self {
+ Self::new(vec![chunk])
+ }
+}
+
+/// A model response: either a live [`Stream`](LlmResponse::Stream) of events or a
/// terminal buffered [`LlmResponse::Agg`] response.
///
/// Not `Clone` — the `Stream` variant owns a single-consumption stream. A buffered
@@ -50,7 +135,7 @@ impl LlmResponse {
}
/// Reduce to the buffered aggregate: return an `Agg` unchanged, or drive a `Stream`
- /// to completion, folding its chunks into an [`AggLlmResponse`] via
+ /// to completion, folding its normalized chunks into an [`AggLlmResponse`] via
/// [`ResponseAccumulator`]. A stream item error aborts with `Err`, as does an
/// in-band [`LlmResponseChunk::DecodeError`] (as `ResponseTranslation`) or
/// [`LlmResponseChunk::StreamError`] (as `UpstreamHttp`).
@@ -60,20 +145,8 @@ impl LlmResponse {
LlmResponse::Stream(mut stream) => {
let mut accumulator = ResponseAccumulator::new();
while let Some(item) = stream.next().await {
- match item? {
- LlmResponseChunk::DecodeError { message } => {
- return Err(LlmClientError::ResponseTranslation(message));
- }
- LlmResponseChunk::StreamError { message } => {
- // The upstream reported the failure inside the response body, so
- // there is no real status line to carry; 502 stands in for "the
- // upstream failed" the same way a failed non-streaming call would.
- return Err(LlmClientError::UpstreamHttp {
- status: MID_STREAM_UPSTREAM_STATUS,
- body: message,
- });
- }
- chunk => accumulator.push(chunk),
+ for chunk in item?.normalized {
+ push_checked_chunk(&mut accumulator, chunk)?;
}
}
Ok(accumulator.finish())
@@ -150,12 +223,32 @@ impl AggLlmResponse {
});
}
chunks.push(LlmResponseChunk::Usage(self.usage));
- Box::pin(futures::stream::iter(chunks.into_iter().map(Ok)))
+ Box::pin(futures::stream::iter(
+ chunks.into_iter().map(|chunk| Ok(chunk.into())),
+ ))
}
}
-/// One provider-neutral streaming event — the normalized counterpart to
-/// [`AggLlmResponse`], sitting between stream decoders and encoders.
+fn push_checked_chunk(
+ accumulator: &mut ResponseAccumulator,
+ chunk: LlmResponseChunk,
+) -> Result<(), LlmClientError> {
+ match chunk {
+ LlmResponseChunk::DecodeError { message } => {
+ Err(LlmClientError::ResponseTranslation(message))
+ }
+ LlmResponseChunk::StreamError { message } => Err(LlmClientError::UpstreamHttp {
+ status: MID_STREAM_UPSTREAM_STATUS,
+ body: message,
+ }),
+ chunk => {
+ accumulator.push(chunk);
+ Ok(())
+ }
+ }
+}
+
+/// One provider-neutral streaming response chunk.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum LlmResponseChunk {
/// Starts a response message.
@@ -397,6 +490,78 @@ mod tests {
);
}
+ #[test]
+ fn aggregates_normalized_chunks_inside_stream_event() {
+ let response = LlmResponse::Stream(Box::pin(stream::iter([Ok(
+ LlmResponseStreamEvent::preserved(
+ crate::WireFormat::OpenAiChat,
+ json!({
+ "choices": [{"delta": {"content": "hello"}}],
+ "system_fingerprint": "fp_exact"
+ }),
+ vec![LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "hello".to_string(),
+ }],
+ ),
+ )])));
+ let aggregate = block_on(response.into_agg()).expect("stream event should aggregate");
+
+ assert_eq!(
+ aggregate.outputs[0].content,
+ vec![ContentBlock::Text {
+ text: "hello".to_string()
+ }]
+ );
+ }
+
+ #[test]
+ fn replacing_normalized_content_drops_preservation() {
+ let event = LlmResponseStreamEvent::preserved(
+ crate::WireFormat::OpenAiChat,
+ json!({"choices": [{"delta": {"content": "old"}}]}),
+ vec![LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "old".to_string(),
+ }],
+ )
+ .replace_normalized(vec![LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "new".to_string(),
+ }]);
+
+ assert!(event.preservation().is_none());
+ assert_eq!(
+ event.normalized(),
+ &[LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "new".to_string(),
+ }]
+ );
+ }
+
+ #[test]
+ fn stream_errors_inside_preserved_events_remain_typed() {
+ let response = LlmResponse::Stream(Box::pin(stream::iter([Ok(
+ LlmResponseStreamEvent::preserved(
+ crate::WireFormat::OpenAiChat,
+ json!({"error": {"message": "provider failed"}}),
+ vec![LlmResponseChunk::StreamError {
+ message: "provider failed".to_string(),
+ }],
+ ),
+ )])));
+
+ let error = block_on(response.into_agg()).err();
+ assert!(matches!(
+ error,
+ Some(LlmClientError::UpstreamHttp {
+ status: MID_STREAM_UPSTREAM_STATUS,
+ ..
+ })
+ ));
+ }
+
#[test]
fn assembles_tool_calls_by_index() {
// id/name arrive once, arguments stream across deltas and parse as JSON.
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
index 56bc44745..8632f2dea 100644
--- a/crates/switchyard-server/src/usage_metrics.rs
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -52,16 +52,22 @@ pub(crate) fn observe(
let wrapped = async_stream::stream! {
let mut latest_usage = None;
while let Some(item) = stream.next().await {
- let failed = matches!(
- &item,
- Err(_)
- | Ok(
+ let failed = match &item {
+ Err(_) => true,
+ Ok(event) => event.normalized().iter().any(|chunk| {
+ matches!(
+ chunk,
LlmResponseChunk::StreamError { .. }
| LlmResponseChunk::DecodeError { .. }
)
- );
- if let Ok(LlmResponseChunk::Usage(usage)) = &item {
- latest_usage = Some(usage.clone());
+ }),
+ };
+ if let Ok(event) = &item {
+ for chunk in event.normalized() {
+ if let LlmResponseChunk::Usage(usage) = chunk {
+ latest_usage = Some(usage.clone());
+ }
+ }
}
if failed {
record_stream_error(&stats, &model, tier.as_deref());
diff --git a/crates/switchyard-translation/src/codecs/anthropic/stream.rs b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
index 0153f8ef9..6cb3d1868 100644
--- a/crates/switchyard-translation/src/codecs/anthropic/stream.rs
+++ b/crates/switchyard-translation/src/codecs/anthropic/stream.rs
@@ -37,6 +37,28 @@ impl StreamCodec for AnthropicMessagesStreamCodec {
encode_anthropic_stream(state, event)
}
+ fn observe_replayed_event(
+ &self,
+ state: &mut StreamTranslationState,
+ raw: &Value,
+ normalized: Vec<LlmResponseChunk>,
+ ) {
+ let normalized_stop = normalized
+ .iter()
+ .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }));
+ for chunk in normalized {
+ drop(encode_anthropic_stream(state, chunk));
+ }
+
+ match raw.get("type").and_then(Value::as_str) {
+ // Anthropic carries the stop reason and usage in `message_delta`, but the separate
+ // `message_stop` event is what actually terminates the stream.
+ Some("message_delta") if normalized_stop => state.emitted_message_delta = true,
+ Some("message_stop") => state.finished = true,
+ _ => {}
+ }
+ }
+
fn finish(&self, state: &mut StreamTranslationState) -> Vec<Value> {
finish_anthropic_stream(state)
}
@@ -189,6 +211,9 @@ fn encode_anthropic_stream(
// Emits any missing Anthropic terminal events and closes open content blocks.
fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
+ if state.finished {
+ return Vec::new();
+ }
let mut out = Vec::new();
if !state.emitted_message_start {
out.extend(encode_anthropic_stream(
@@ -227,14 +252,17 @@ fn finish_anthropic_stream(state: &mut StreamTranslationState) -> Vec<Value> {
out.push(json!({"type": "content_block_stop", "index": 0}));
}
- out.push(json!({
- "type": "message_delta",
- "delta": {
- "stop_reason": anthropic_stop_reason(state.stop_reason.as_deref()),
- "stop_sequence": Value::Null,
- },
- "usage": anthropic_stream_usage(state),
- }));
+ if !state.emitted_message_delta {
+ out.push(json!({
+ "type": "message_delta",
+ "delta": {
+ "stop_reason": anthropic_stop_reason(state.stop_reason.as_deref()),
+ "stop_sequence": Value::Null,
+ },
+ "usage": anthropic_stream_usage(state),
+ }));
+ state.emitted_message_delta = true;
+ }
out.push(json!({"type": "message_stop"}));
state.finished = true;
out
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index b11922e07..5ea2f10fd 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -156,6 +156,9 @@ fn encode_openai_chat_stream(
match event {
LlmResponseChunk::MessageStart { id, model } => {
record_source_identity(state, id, model);
+ // `finish_openai_chat_stream` uses this flag to decide whether a clean EOF needs a
+ // synthesized terminal chunk. Set it on the encode path as well as the decode path.
+ state.saw_message_start = true;
if state.emitted_message_start
|| (!state_source_is(state, WireFormat::AnthropicMessages)
&& !state_source_is(state, WireFormat::OpenAiResponses))
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 31b269ce2..10bf323f3 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -185,6 +185,9 @@ fn encode_responses_stream(
// Emits final OpenAI Responses completion events from accumulated state.
fn finish_responses_stream(state: &mut StreamTranslationState) -> Vec<Value> {
+ if state.finished {
+ return Vec::new();
+ }
let mut out = ensure_responses_created(state);
if state.response_text_started
&& let Some(output_index) = state.response_text_output_index
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index 318be86e8..79782907c 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -40,6 +40,7 @@ pub struct StreamTranslationState {
pub(crate) output_tokens_seen: u64,
pub(crate) saw_backend_usage: bool,
pub(crate) stop_reason: Option<String>,
+ pub(crate) emitted_message_delta: bool,
pub(crate) next_content_index: usize,
pub(crate) text_block_index: Option<usize>,
@@ -109,6 +110,30 @@ pub trait StreamCodec: Send + Sync {
event: LlmResponseChunk,
) -> Vec<Value>;
+ /// Advances encoder state after an exact same-format event replay.
+ ///
+ /// Exact replay returns the preserved provider JSON instead of the JSON emitted by
+ /// [`Self::encode_event`]. The encoder must nevertheless observe the normalized chunks so
+ /// [`Self::finish`] can close an incomplete stream without duplicating an already replayed
+ /// terminal event. Codecs whose terminal state cannot be inferred from `MessageStop` alone
+ /// may override this hook and inspect `raw`.
+ fn observe_replayed_event(
+ &self,
+ state: &mut StreamTranslationState,
+ _raw: &Value,
+ normalized: Vec<LlmResponseChunk>,
+ ) {
+ let replayed_terminal = normalized
+ .iter()
+ .any(|chunk| matches!(chunk, LlmResponseChunk::MessageStop { .. }));
+ for chunk in normalized {
+ drop(self.encode_event(state, chunk));
+ }
+ if replayed_terminal {
+ state.finished = true;
+ }
+ }
+
/// Emits any terminal provider events needed after the source stream ends.
///
/// This is intentionally required on every codec. Some target formats
@@ -215,6 +240,31 @@ pub fn decode_stream_event(
})
}
+pub(crate) fn encode_response_stream_event(
+ state: &mut StreamTranslationState,
+ target_codec: &dyn StreamCodec,
+ target: &FormatId,
+ event: crate::LlmResponseStreamEvent,
+) -> Vec<Value> {
+ let (preservation, normalized) = event.into_parts();
+ if let Some(preservation) = preservation {
+ let (source, raw) = preservation.into_parts();
+ if &source == target {
+ // Exact replay bypasses the target encoder's emitted JSON, but the encoder must
+ // still observe every normalized chunk. Otherwise `finish` starts from empty state:
+ // a clean EOF after a nonterminal provider event can omit or synthesize malformed
+ // terminal events, while a replayed terminal can be emitted twice.
+ target_codec.observe_replayed_event(state, &raw, normalized);
+ return vec![raw];
+ }
+ }
+
+ normalized
+ .into_iter()
+ .flat_map(|chunk| target_codec.encode_event(state, chunk))
+ .collect()
+}
+
/// Encodes one neutral stream event with the built-in codec registry.
pub fn encode_stream_event(
state: &mut StreamTranslationState,
diff --git a/crates/switchyard-translation/src/engine.rs b/crates/switchyard-translation/src/engine.rs
index e7f646d9d..9761a46af 100644
--- a/crates/switchyard-translation/src/engine.rs
+++ b/crates/switchyard-translation/src/engine.rs
@@ -8,11 +8,14 @@ use std::sync::Arc;
use serde_json::Value;
+use crate::LlmResponseStreamEvent;
use crate::codecs::FormatCodec;
use crate::codecs::anthropic::AnthropicMessagesCodec;
use crate::codecs::openai_chat::OpenAiChatCodec;
use crate::codecs::responses::OpenAiResponsesCodec;
-use crate::codecs::stream::{StreamCodecRegistry, StreamTranslationState};
+use crate::codecs::stream::{
+ StreamCodecRegistry, StreamTranslationState, encode_response_stream_event,
+};
use crate::diagnostic::TranslationDiagnostic;
use crate::error::{Result, TranslationError};
use crate::format::FormatId;
@@ -243,6 +246,45 @@ impl TranslationEngine {
.collect())
}
+ /// Decodes one provider event while retaining its parsed source JSON value.
+ ///
+ /// Takes ownership of `event` so preservation does not deep-copy provider JSON
+ /// on the per-event streaming path.
+ pub fn decode_stream_event(
+ &self,
+ state: &mut StreamTranslationState,
+ source: impl Into<FormatId>,
+ event: Value,
+ ) -> Result<LlmResponseStreamEvent> {
+ let source = source.into();
+ let source_codec = self.stream_registry.codec(source.clone())?;
+ state.source = Some(source.clone());
+ let normalized = source_codec.decode_event(state, &event);
+ Ok(LlmResponseStreamEvent::preserved(source, event, normalized))
+ }
+
+ /// Encodes one neutral or preserved stream event for a target provider.
+ ///
+ /// A preserved event replays its retained JSON value unchanged when its
+ /// source and target formats match. Cross-format encoding intentionally
+ /// uses only its normalized events.
+ pub fn encode_stream_event(
+ &self,
+ state: &mut StreamTranslationState,
+ target: impl Into<FormatId>,
+ event: LlmResponseStreamEvent,
+ ) -> Result<Vec<Value>> {
+ let target = target.into();
+ let target_codec = self.stream_registry.codec(target.clone())?;
+ state.target = Some(target.clone());
+ Ok(encode_response_stream_event(
+ state,
+ target_codec.as_ref(),
+ &target,
+ event,
+ ))
+ }
+
/// Finishes target-provider stream emission after the source stream closes.
pub fn finish_stream(
&self,
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index dbb047514..5136825c8 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -15,10 +15,11 @@ use futures::{Stream, StreamExt, TryStreamExt};
use serde_json::Value;
use switchyard_protocol::LlmClientError;
+use crate::codecs::stream::encode_response_stream_event;
use crate::sse;
use crate::{
- AggLlmResponse, LlmRequest, LlmResponseStream, Result, StreamCodecRegistry,
- StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat,
+ AggLlmResponse, FormatId, LlmRequest, LlmResponseStream, LlmResponseStreamEvent, Result,
+ StreamCodecRegistry, StreamTranslationState, TranslationEngine, TranslationPolicy, WireFormat,
};
static DEFAULT_TRANSLATION_POLICY: LazyLock<TranslationPolicy> =
@@ -85,17 +86,19 @@ pub fn encode_stream(
target: WireFormat,
served_model: Option<String>,
) -> std::result::Result<RawEventStream, LlmClientError> {
+ let target_format: FormatId = target.into();
// The target is always a built-in wire format, so this lookup cannot fail; a
// failure returns as an `Err` rather than a panic.
let codec = StreamCodecRegistry::with_builtins()
- .codec(target)
+ .codec(target_format.clone())
// Currently the only error is that the codec is missing, which is Configuration
.map_err(|err| LlmClientError::Configuration {
message: err.to_string(),
})?;
+ let served_model_for_events = served_model.clone();
let mut state = StreamTranslationState {
- target: Some(target.into()),
+ target: Some(target_format.clone()),
target_model: served_model,
..Default::default()
};
@@ -103,12 +106,24 @@ pub fn encode_stream(
let events = try_stream! {
while let Some(item) = chunks.next().await {
- let chunk = item?;
- for value in codec.encode_event(&mut state, chunk) {
+ let event = item?;
+ for mut value in
+ encode_response_stream_event(&mut state, codec.as_ref(), &target_format, event)
+ {
+ stamp_streamed_response_model(
+ &mut value,
+ target,
+ served_model_for_events.as_deref(),
+ );
yield value;
}
}
- for value in codec.finish(&mut state) {
+ for mut value in codec.finish(&mut state) {
+ stamp_streamed_response_model(
+ &mut value,
+ target,
+ served_model_for_events.as_deref(),
+ );
yield value;
}
};
@@ -116,6 +131,37 @@ pub fn encode_stream(
Ok(Box::pin(events))
}
+// The raw-response helper promises that the caller sees the model that served the
+// request. Same-format preservation bypasses provider codecs, so apply that
+// helper-specific override after replay without disturbing any other raw fields.
+fn stamp_streamed_response_model(
+ event: &mut Value,
+ target: WireFormat,
+ served_model: Option<&str>,
+) {
+ let Some(served_model) = served_model else {
+ return;
+ };
+
+ match target {
+ WireFormat::OpenAiChat => {
+ if let Some(event) = event.as_object_mut() {
+ event.insert("model".to_string(), Value::String(served_model.to_string()));
+ }
+ }
+ WireFormat::OpenAiResponses => {
+ if let Some(response) = event.get_mut("response").and_then(Value::as_object_mut) {
+ response.insert("model".to_string(), Value::String(served_model.to_string()));
+ }
+ }
+ WireFormat::AnthropicMessages => {
+ if let Some(message) = event.get_mut("message").and_then(Value::as_object_mut) {
+ message.insert("model".to_string(), Value::String(served_model.to_string()));
+ }
+ }
+ }
+}
+
/// Decodes a byte stream of `source`-format SSE frames into neutral IR chunks.
///
/// Operates on raw bytes, not any HTTP client type: the caller adapts its
@@ -130,10 +176,11 @@ where
S: Stream<Item = std::result::Result<Vec<u8>, LlmClientError>> + Send + 'static,
{
let marker = sse::done_marker(source);
+ let source_format: FormatId = source.into();
// The source is always a built-in wire format, so this lookup cannot fail; a
// failure returns as an `Err` rather than a panic.
let codec = StreamCodecRegistry::with_builtins()
- .codec(source)
+ .codec(source_format.clone())
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
// Adapt the byte-chunk stream into an async line reader. The BufReader
// reassembles data split across network chunks (including multi-byte UTF-8),
@@ -145,7 +192,10 @@ where
Box::pin(bytes.map(|item| item.map_err(std::io::Error::other)));
let lines = futures::io::BufReader::new(io_bytes.into_async_read()).lines();
- let mut state = StreamTranslationState::default();
+ let mut state = StreamTranslationState {
+ source: Some(source_format.clone()),
+ ..StreamTranslationState::default()
+ };
let mut frame = String::new();
let stream = Box::pin(try_stream! {
futures::pin_mut!(lines);
@@ -160,9 +210,12 @@ where
sse::SseFrame::Empty => {}
sse::SseFrame::Done => break,
sse::SseFrame::Data(value) => {
- for event in codec.decode_event(&mut state, &value) {
- yield event;
- }
+ let normalized = codec.decode_event(&mut state, &value);
+ yield LlmResponseStreamEvent::preserved(
+ source_format.clone(),
+ value,
+ normalized,
+ );
}
}
} else {
@@ -178,9 +231,8 @@ where
let parsed = sse::parse_json_sse_frame(&frame, marker)
.map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
if let sse::SseFrame::Data(value) = parsed {
- for event in codec.decode_event(&mut state, &value) {
- yield event;
- }
+ let normalized = codec.decode_event(&mut state, &value);
+ yield LlmResponseStreamEvent::preserved(source_format, value, normalized);
}
}
});
@@ -208,11 +260,13 @@ mod tests {
use futures::executor::block_on;
use futures::{Stream, StreamExt, stream};
use serde_json::{Value, json};
- use switchyard_protocol::{LlmClientError, LlmResponseChunk, completion_text};
+ use switchyard_protocol::{
+ LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, completion_text,
+ };
use super::{
decode_aggregated_response, decode_request, decode_stream, encode_aggregated_response,
- encode_request, encode_stream,
+ encode_request, encode_stream, stamp_streamed_response_model,
};
use crate::{LlmResponseStream, WireFormat};
@@ -223,19 +277,23 @@ mod tests {
fn decode_all(
bytes: impl Stream<Item = Result<Vec<u8>, LlmClientError>> + Send + 'static,
source: WireFormat,
- ) -> Result<Vec<LlmResponseChunk>, LlmClientError> {
+ ) -> Result<Vec<LlmResponseStreamEvent>, LlmClientError> {
block_on(decode_stream(bytes, source)?.collect::<Vec<_>>())
.into_iter()
.collect()
}
// Concatenates the text of every `TextDelta` chunk.
- fn text_of(chunks: &[LlmResponseChunk]) -> String {
- chunks
+ fn text_of(events: &[LlmResponseStreamEvent]) -> String {
+ events
.iter()
- .filter_map(|chunk| match chunk {
- LlmResponseChunk::TextDelta { text, .. } => Some(text.as_str()),
- _ => None,
+ .flat_map(LlmResponseStreamEvent::normalized)
+ .filter_map(|chunk| {
+ if let LlmResponseChunk::TextDelta { text, .. } = chunk {
+ Some(text.as_str())
+ } else {
+ None
+ }
})
.collect()
}
@@ -281,14 +339,17 @@ mod tests {
Ok(LlmResponseChunk::TextDelta {
index: 0,
text: "Hello".to_string(),
- }),
+ }
+ .into()),
Ok(LlmResponseChunk::TextDelta {
index: 0,
text: " world".to_string(),
- }),
+ }
+ .into()),
Ok(LlmResponseChunk::MessageStop {
reason: Some("stop".to_string()),
- }),
+ }
+ .into()),
])
.boxed();
@@ -321,11 +382,13 @@ mod tests {
Ok(LlmResponseChunk::MessageStart {
id: Some("msg_1".to_string()),
model: Some("upstream/model".to_string()),
- }),
+ }
+ .into()),
Ok(LlmResponseChunk::TextDelta {
index: 0,
text: "hi".to_string(),
- }),
+ }
+ .into()),
])
.boxed();
@@ -353,11 +416,13 @@ mod tests {
Ok(LlmResponseChunk::MessageStart {
id: Some("msg_1".to_string()),
model: Some("upstream/model".to_string()),
- }),
+ }
+ .into()),
Ok(LlmResponseChunk::TextDelta {
index: 0,
text: "hi".to_string(),
- }),
+ }
+ .into()),
])
.boxed();
@@ -374,7 +439,7 @@ mod tests {
#[test]
fn encode_stream_propagates_chunk_errors() -> Result<(), BoxError> {
let chunks: LlmResponseStream =
- stream::iter(vec![Err::<LlmResponseChunk, LlmClientError>(
+ stream::iter(vec![Err::<LlmResponseStreamEvent, LlmClientError>(
LlmClientError::General("chunk exploded".to_string()),
)])
.boxed();
@@ -397,6 +462,87 @@ mod tests {
Ok(())
}
+ #[test]
+ fn stream_helpers_replay_same_format_provider_fields() -> Result<(), BoxError> {
+ let provider_event = json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "system_fingerprint": "fp_provider_specific",
+ "choices": [{
+ "index": 0,
+ "delta": {"content": "Hello"},
+ "finish_reason": "stop"
+ }]
+ });
+ let bytes = stream::once({
+ let frame = format!("data: {provider_event}\n\n").into_bytes();
+ async move { Ok::<Vec<u8>, LlmClientError>(frame) }
+ });
+ let decoded = decode_stream(bytes, WireFormat::OpenAiChat)?;
+ let replayed =
+ block_on(encode_stream(decoded, WireFormat::OpenAiChat, None)?.collect::<Vec<_>>())
+ .into_iter()
+ .collect::<Result<Vec<Value>, BoxError>>()?;
+
+ assert_eq!(replayed, vec![provider_event]);
+ Ok(())
+ }
+
+ #[test]
+ fn openai_chat_replay_stamps_the_served_model_without_losing_extensions() {
+ let mut event = json!({
+ "choices": [{"delta": {"content": "Hello"}}],
+ "system_fingerprint": "fp_provider_specific",
+ });
+
+ stamp_streamed_response_model(&mut event, WireFormat::OpenAiChat, Some("served/model"));
+
+ assert_eq!(event["model"], "served/model");
+ assert_eq!(event["system_fingerprint"], "fp_provider_specific");
+ }
+
+ #[test]
+ fn responses_replay_stamps_the_served_model_inside_response() {
+ let mut event = json!({
+ "type": "response.created",
+ "response": {
+ "id": "resp_1",
+ "model": "provider/model",
+ "provider_extension": true,
+ },
+ });
+
+ stamp_streamed_response_model(
+ &mut event,
+ WireFormat::OpenAiResponses,
+ Some("served/model"),
+ );
+
+ assert_eq!(event["response"]["model"], "served/model");
+ assert_eq!(event["response"]["provider_extension"], true);
+ }
+
+ #[test]
+ fn anthropic_replay_stamps_the_served_model_inside_message() {
+ let mut event = json!({
+ "type": "message_start",
+ "message": {
+ "id": "msg_1",
+ "model": "provider/model",
+ "provider_extension": true,
+ },
+ });
+
+ stamp_streamed_response_model(
+ &mut event,
+ WireFormat::AnthropicMessages,
+ Some("served/model"),
+ );
+
+ assert_eq!(event["message"]["model"], "served/model");
+ assert_eq!(event["message"]["provider_extension"], true);
+ }
+
#[test]
fn decode_stream_reassembles_frames_split_across_chunks() -> Result<(), BoxError> {
// A multi-byte codepoint and the frame boundaries are split across
diff --git a/crates/switchyard-translation/src/lib.rs b/crates/switchyard-translation/src/lib.rs
index 5cdc51d98..d07a3497d 100644
--- a/crates/switchyard-translation/src/lib.rs
+++ b/crates/switchyard-translation/src/lib.rs
@@ -17,8 +17,9 @@ mod sse;
pub mod stream;
pub mod util;
-pub use switchyard_protocol::stream::LlmResponseChunk;
-pub use switchyard_protocol::stream::LlmResponseStream;
+pub use switchyard_protocol::stream::{
+ LlmResponseChunk, LlmResponseStream, LlmResponseStreamEvent, ProviderStreamEvent,
+};
pub use switchyard_protocol::{format, llm};
pub use diagnostic::*;
diff --git a/crates/switchyard-translation/tests/extension_points.rs b/crates/switchyard-translation/tests/extension_points.rs
index 3ba377fcd..d7db6f41d 100644
--- a/crates/switchyard-translation/tests/extension_points.rs
+++ b/crates/switchyard-translation/tests/extension_points.rs
@@ -86,6 +86,26 @@ fn custom_stream_codec_can_participate_in_registered_stream_translation() -> Tes
Ok(())
}
+#[test]
+fn custom_stream_codec_replays_preserved_same_format_event() -> TestResult {
+ let mut registry = StreamCodecRegistry::new();
+ registry.register(CustomStreamCodec);
+ let engine = TranslationEngine::with_registries(FormatRegistry::new(), registry);
+ let format = FormatId::new("custom_stream");
+ let mut state = StreamTranslationState::new(format.clone(), format.clone());
+ let event = json!({
+ "kind": "delta",
+ "text": "hello",
+ "vendor_only": {"must": "survive"}
+ });
+
+ let preserved = engine.decode_stream_event(&mut state, format.clone(), event.clone())?;
+ let replayed = engine.encode_stream_event(&mut state, format, preserved)?;
+
+ assert_eq!(replayed, vec![event]);
+ Ok(())
+}
+
// Verifies target capability policy can reject unsupported request features.
#[test]
fn capability_profile_can_fail_fast_when_target_cannot_accept_request_features() {
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 38a6f628c..d97eee149 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -12,6 +12,222 @@ use switchyard_translation::{
type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;
+// Same-format replay returns the same parsed JSON value, including provider-specific fields.
+#[test]
+fn preserved_same_format_events_replay_unknown_fields_exactly() -> TestResult {
+ let cases = [
+ (
+ WireFormat::OpenAiChat,
+ json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "system_fingerprint": "fp_provider_specific",
+ "choices": [{
+ "index": 0,
+ "delta": {"content": "Hi"},
+ "finish_reason": null
+ }]
+ }),
+ ),
+ (
+ WireFormat::OpenAiResponses,
+ json!({
+ "type": "response.output_text.delta",
+ "item_id": "item-1",
+ "output_index": 0,
+ "content_index": 0,
+ "delta": "Hi",
+ "sequence_number": 2,
+ "provider_extension": {"exact": true}
+ }),
+ ),
+ (
+ WireFormat::AnthropicMessages,
+ json!({
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {"type": "text_delta", "text": "Hi"},
+ "provider_extension": {"exact": true}
+ }),
+ ),
+ ];
+
+ for (format, event) in cases {
+ let engine = TranslationEngine::default();
+ let mut state = StreamTranslationState::new(format, format);
+ let preserved = engine.decode_stream_event(&mut state, format, event.clone())?;
+ let replayed = engine.encode_stream_event(&mut state, format, preserved)?;
+ assert_eq!(replayed, vec![event]);
+ }
+ Ok(())
+}
+
+// Replay emits the preserved event without running the encoder, so the encoder never sees the
+// stop it would normally record. Replay must still leave the stream marked finished or
+// `finish_stream` synthesizes a terminal the client already received.
+#[test]
+fn replayed_terminal_event_suppresses_synthesized_finish() -> TestResult {
+ let cases = [
+ (
+ WireFormat::OpenAiChat,
+ vec![
+ json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "choices": [{"index": 0, "delta": {"content": "Hi"}, "finish_reason": null}]
+ }),
+ json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+ }),
+ ],
+ ),
+ (
+ WireFormat::AnthropicMessages,
+ vec![
+ json!({"type": "message_start", "message": {"id": "msg_1", "model": "claude"}}),
+ json!({"type": "message_stop"}),
+ ],
+ ),
+ (
+ WireFormat::OpenAiResponses,
+ vec![
+ json!({"type": "response.created", "response": {"id": "resp_1", "model": "gpt-4o"}}),
+ json!({"type": "response.completed", "response": {"id": "resp_1", "model": "gpt-4o"}}),
+ ],
+ ),
+ ];
+
+ let engine = TranslationEngine::default();
+ for (format, events) in cases {
+ let mut state = StreamTranslationState::new(format, format);
+ let mut replayed = Vec::new();
+ for event in &events {
+ let preserved = engine.decode_stream_event(&mut state, format, event.clone())?;
+ replayed.extend(engine.encode_stream_event(&mut state, format, preserved)?);
+ }
+ assert_eq!(replayed, events, "{format:?} engine replay");
+ assert!(
+ engine.finish_stream(&mut state, format)?.is_empty(),
+ "{format:?} engine replay already delivered a terminal event",
+ );
+ }
+ Ok(())
+}
+
+#[test]
+fn replayed_nonterminal_event_advances_encoder_state_before_finish() -> TestResult {
+ let engine = TranslationEngine::default();
+ let format = WireFormat::OpenAiChat;
+ let event = json!({
+ "id": "chatcmpl-clean-eof",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "choices": [{
+ "index": 0,
+ "delta": {"role": "assistant", "content": "Hi"},
+ "finish_reason": null
+ }]
+ });
+ let mut decode_state = StreamTranslationState::new(format, format);
+ let preserved = engine.decode_stream_event(&mut decode_state, format, event.clone())?;
+
+ // Provider decoding and caller encoding are independent stream boundaries in a host such as
+ // NeMo Relay. Replay must therefore advance a fresh encoder state using the normalized view.
+ let mut encode_state = StreamTranslationState::new(format, format);
+ assert_eq!(
+ engine.encode_stream_event(&mut encode_state, format, preserved)?,
+ vec![event]
+ );
+ let finish = engine.finish_stream(&mut encode_state, format)?;
+
+ assert_eq!(finish.len(), 1);
+ assert_eq!(finish[0]["id"], "chatcmpl-clean-eof");
+ assert_eq!(finish[0]["model"], "gpt-4o");
+ assert_eq!(finish[0]["choices"][0]["finish_reason"], "stop");
+ Ok(())
+}
+
+#[test]
+fn replayed_anthropic_terminal_delta_finishes_with_message_stop_only() -> TestResult {
+ let engine = TranslationEngine::default();
+ let format = WireFormat::AnthropicMessages;
+ let events = [
+ json!({
+ "type": "message_start",
+ "message": {
+ "id": "msg_clean_eof",
+ "model": "claude",
+ "usage": {"input_tokens": 3, "output_tokens": 0}
+ }
+ }),
+ json!({
+ "type": "message_delta",
+ "delta": {"stop_reason": "end_turn", "stop_sequence": null},
+ "usage": {"output_tokens": 2}
+ }),
+ ];
+ let mut decode_state = StreamTranslationState::new(format, format);
+ let mut encode_state = StreamTranslationState::new(format, format);
+
+ for event in events {
+ let preserved = engine.decode_stream_event(&mut decode_state, format, event.clone())?;
+ assert_eq!(
+ engine.encode_stream_event(&mut encode_state, format, preserved)?,
+ vec![event]
+ );
+ }
+
+ let finish = engine.finish_stream(&mut encode_state, format)?;
+ assert_eq!(
+ finish
+ .iter()
+ .filter(|event| event["type"] == "message_stop")
+ .count(),
+ 1
+ );
+ assert!(
+ finish.iter().all(|event| event["type"] != "message_delta"),
+ "the replayed terminal delta must not be synthesized a second time"
+ );
+ Ok(())
+}
+
+// Cross-format encoding discards provider-specific fields and uses normalized chunks.
+#[test]
+fn preserved_cross_format_event_uses_normalized_content() -> TestResult {
+ let engine = TranslationEngine::default();
+ let mut state =
+ StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::AnthropicMessages);
+ let event = json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "system_fingerprint": "fp_provider_specific",
+ "choices": [{
+ "index": 0,
+ "delta": {"content": "Hi"},
+ "finish_reason": null
+ }]
+ });
+
+ let preserved = engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, event)?;
+ let translated =
+ engine.encode_stream_event(&mut state, WireFormat::AnthropicMessages, preserved)?;
+
+ assert_eq!(translated[2]["delta"]["text"], "Hi");
+ assert!(
+ translated
+ .iter()
+ .all(|event| event.get("system_fingerprint").is_none())
+ );
+ Ok(())
+}
+
// Verifies an OpenAI text delta opens the expected Anthropic message and content blocks.
#[test]
fn openai_chat_stream_event_translates_to_anthropic_message_events() -> TestResult {