Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): emit errors for same-wire stream decode failures (#747)

Graham Kingb7b13c6ef53e
diff --git a/crates/switchyard-translation/src/codecs/stream.rs b/crates/switchyard-translation/src/codecs/stream.rs
index a0dfc6a55..673e68258 100644
--- a/crates/switchyard-translation/src/codecs/stream.rs
+++ b/crates/switchyard-translation/src/codecs/stream.rs
@@ -300,7 +300,11 @@ pub(crate) fn encode_response_stream_event(
let (preservation, normalized) = event.into_parts();
if let Some(preservation) = preservation {
let (source, raw) = preservation.into_parts();
- if &source == target {
+ // Invalid protocol data must become an error frame, not be replayed as ordinary data.
+ let has_decode_error = normalized
+ .iter()
+ .any(|chunk| matches!(chunk, LlmResponseChunk::DecodeError { .. }));
+ if &source == target && !has_decode_error {
// Exact replay bypasses the target encoder's emitted JSON, but the encoder must
// still observe every normalized chunk. Otherwise `finish` starts from empty state:
// a clean EOF after a nonterminal provider event can omit or synthesize malformed
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index 9d5425803..c9de9b8d7 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -141,6 +141,53 @@ fn preserved_same_format_replay_stops_after_an_error() -> TestResult {
Ok(())
}
+#[test]
+fn same_format_decode_failure_emits_terminal_error() -> TestResult {
+ let cases = [
+ (
+ WireFormat::OpenAiChat,
+ json!({"choices": [{"index": 0, "delta": {"content": "partial"}}]}),
+ ),
+ (
+ WireFormat::AnthropicMessages,
+ json!({
+ "type": "content_block_delta", "index": 0,
+ "delta": {"type": "text_delta", "text": "partial"}
+ }),
+ ),
+ ];
+ let engine = TranslationEngine::default();
+ for (format, text) in cases {
+ let mut decode_state = StreamTranslationState::new(format, format);
+ let mut encode_state = StreamTranslationState::new(format, format);
+ let preserved = engine.decode_stream_event(&mut decode_state, format, text.clone())?;
+ assert_eq!(
+ engine.encode_stream_event(&mut encode_state, format, preserved)?,
+ vec![text.clone()]
+ );
+
+ let invalid = engine.decode_stream_event(&mut decode_state, format, Value::Null)?;
+ let errors = engine.encode_stream_event(&mut encode_state, format, invalid)?;
+ assert_eq!(errors.len(), 1, "{format:?}");
+ assert!(
+ errors[0]["error"]["message"].is_string(),
+ "{format:?}: {errors:?}"
+ );
+ if format == WireFormat::AnthropicMessages {
+ assert_eq!(errors[0]["type"], "error");
+ }
+
+ let later = engine.decode_stream_event(&mut decode_state, format, text)?;
+ assert!(
+ engine
+ .encode_stream_event(&mut encode_state, format, later)?
+ .is_empty()
+ );
+ assert!(engine.finish_stream(&mut encode_state, format)?.is_empty());
+ }
+ Ok(())
+}
+
// Replay emits the preserved event without running the encoder, so the encoder never sees the
// stop it would normally record. Replay must still leave the stream marked finished or
// `finish_stream` synthesizes a terminal the client already received.