Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(translation): retain response IDs arriving after stream start (#771)
nachiketb-nvidia2be08d303a78
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 8fd07a166..4024658e1 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -1243,22 +1243,21 @@ struct StreamingUsageClient;#[async_trait]impl RoutedLlmClient for StreamingUsageClient {async fn call(&self, request: Request) -> Result<Response, LlmClientError> {- let usage = Usage {- input_tokens: Some(13),- output_tokens: Some(5),- cache: Usage::cache_details(Some(8), None),- ..Usage::default()- };- let chunks = vec![Ok(LlmResponseStreamEvent::new(vec![- LlmResponseChunk::MessageStart {- id: Some("obs-stream-response".to_string()),- model: request.model_id().map(|s| s.to_string()),- },- LlmResponseChunk::Usage(usage),- LlmResponseChunk::MessageStop {- reason: Some("end_turn".to_string()),- },- ]))];+ let engine = switchyard_translation::TranslationEngine::default();+ let mut state = switchyard_translation::StreamTranslationState::new(+ WireFormat::OpenAiChat,+ WireFormat::OpenAiChat,+ );+ let chunks = [+ json!({"id": "", "model": request.model_id(), "choices": []}),+ json!({"id": "", "choices": [{"index": 0, "delta": {"content": "hello"}}]}),+ json!({"id": "obs-stream-response", "choices": []}),+ json!({"id": "obs-stream-response", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 21, "completion_tokens": 5, "prompt_tokens_details": {"cached_tokens": 8}}}),+ ].into_iter().map(|event| {+ Ok(engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, event)+ .expect("valid Chat fixture"))+ }).collect::<Vec<_>>();Ok(Response {llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))),metadata: None,@@ -1291,11 +1290,41 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resulet LlmResponse::Stream(mut stream) = response.llm_response else {return Err(test_error("expected a streamed response"));};+ let engine = switchyard_translation::TranslationEngine::default();+ let mut translated = switchyard_translation::StreamTranslationState::new(+ WireFormat::OpenAiChat,+ WireFormat::OpenAiResponses,+ );+ let mut events = Vec::new();while let Some(item) = stream.next().await {- if let Err(error) = item {- panic!("unexpected stream error: {error}");- }+ events.extend(+ engine+ .encode_stream_event(+ &mut translated,+ WireFormat::OpenAiResponses,+ item.expect("valid stream"),+ )+ .expect("valid translation"),+ );}+ events.extend(+ engine+ .finish_stream(&mut translated, WireFormat::OpenAiResponses)+ .expect("valid finish"),+ );+ let created = events+ .iter()+ .find(|e| e["type"] == "response.created")+ .expect("created");+ let completed = events+ .iter()+ .find(|e| e["type"] == "response.completed")+ .expect("completed");+ assert_eq!(created["response"]["id"], completed["response"]["id"]);+ assert_eq!(+ completed["response"]["output"][0]["content"][0]["text"],+ "hello"+ );let spans = store.spans();let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);@@ -1318,7 +1347,7 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resuassert!(matches!(otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),Some(OtelValue::Array(OtelArray::String(reasons)))- if reasons.len() == 1 && reasons[0].as_str() == "end_turn"+ if reasons.len() == 1 && reasons[0].as_str() == "stop"));Ok(())}diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rsindex 37bf123c8..c59a66f77 100644--- a/crates/protocol/src/stream.rs+++ b/crates/protocol/src/stream.rs@@ -297,7 +297,7 @@ fn push_checked_chunk(/// One provider-neutral streaming response chunk.#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]pub enum LlmResponseChunk {- /// Starts a response message.+ /// Starts a response message, or fills in identity first supplied by a later frame.MessageStart {/// Provider response identifier.id: Option<String>,diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rsindex e87e21728..86fdf124d 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs@@ -67,14 +67,17 @@ fn decode_openai_chat_stream(}let mut out = Vec::new();- if !state.saw_message_start {+ let mut identity_changed = false;+ if state.model.is_none() {+ state.model = string_field(object, "model");+ identity_changed |= state.model.is_some();+ }+ if state.message_id.is_none() {+ state.message_id = string_field(object, "id");+ identity_changed |= state.message_id.is_some();+ }+ if !state.saw_message_start || identity_changed {state.saw_message_start = true;- if let Some(model) = string_field(object, "model") {- state.model = Some(model);- }- if let Some(id) = string_field(object, "id") {- state.message_id = Some(id);- }out.push(LlmResponseChunk::MessageStart {id: state.message_id.clone(),model: state.model.clone(),diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rsindex 399b0c3e2..7aa6df092 100644--- a/crates/switchyard-translation/src/codecs/responses/stream.rs+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs@@ -732,6 +732,10 @@ fn ensure_responses_created(state: &mut StreamTranslationState) -> Vec<Value> {if state.response_created {return Vec::new();}+ // A provider ID learned later must not change an already published response or item ID.+ if state.target_message_id.is_none() {+ state.target_message_id = Some(responses_id(state));+ }state.response_created = true;vec![json!({"type": "response.created",