Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(translation): retain response IDs arriving after stream start (#771)

nachiketb-nvidia2be08d303a78
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 8fd07a166..4024658e1 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -1243,22 +1243,21 @@ struct StreamingUsageClient;
#[async_trait]
impl RoutedLlmClient for StreamingUsageClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
- let usage = Usage {
- input_tokens: Some(13),
- output_tokens: Some(5),
- cache: Usage::cache_details(Some(8), None),
- ..Usage::default()
- };
- let chunks = vec![Ok(LlmResponseStreamEvent::new(vec![
- LlmResponseChunk::MessageStart {
- id: Some("obs-stream-response".to_string()),
- model: request.model_id().map(|s| s.to_string()),
- },
- LlmResponseChunk::Usage(usage),
- LlmResponseChunk::MessageStop {
- reason: Some("end_turn".to_string()),
- },
- ]))];
+ let engine = switchyard_translation::TranslationEngine::default();
+ let mut state = switchyard_translation::StreamTranslationState::new(
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiChat,
+ );
+ let chunks = [
+ json!({"id": "", "model": request.model_id(), "choices": []}),
+ json!({"id": "", "choices": [{"index": 0, "delta": {"content": "hello"}}]}),
+ json!({"id": "obs-stream-response", "choices": []}),
+ json!({"id": "obs-stream-response", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 21, "completion_tokens": 5, "prompt_tokens_details": {"cached_tokens": 8}}}),
+ ].into_iter().map(|event| {
+ Ok(engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, event)
+ .expect("valid Chat fixture"))
+ }).collect::<Vec<_>>();
Ok(Response {
llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))),
metadata: None,
@@ -1291,11 +1290,41 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resu
let LlmResponse::Stream(mut stream) = response.llm_response else {
return Err(test_error("expected a streamed response"));
};
+ let engine = switchyard_translation::TranslationEngine::default();
+ let mut translated = switchyard_translation::StreamTranslationState::new(
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiResponses,
+ );
+ let mut events = Vec::new();
while let Some(item) = stream.next().await {
- if let Err(error) = item {
- panic!("unexpected stream error: {error}");
- }
+ events.extend(
+ engine
+ .encode_stream_event(
+ &mut translated,
+ WireFormat::OpenAiResponses,
+ item.expect("valid stream"),
+ )
+ .expect("valid translation"),
+ );
}
+ events.extend(
+ engine
+ .finish_stream(&mut translated, WireFormat::OpenAiResponses)
+ .expect("valid finish"),
+ );
+ let created = events
+ .iter()
+ .find(|e| e["type"] == "response.created")
+ .expect("created");
+ let completed = events
+ .iter()
+ .find(|e| e["type"] == "response.completed")
+ .expect("completed");
+ assert_eq!(created["response"]["id"], completed["response"]["id"]);
+ assert_eq!(
+ completed["response"]["output"][0]["content"][0]["text"],
+ "hello"
+ );
let spans = store.spans();
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
@@ -1318,7 +1347,7 @@ async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Resu
assert!(matches!(
otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),
Some(OtelValue::Array(OtelArray::String(reasons)))
- if reasons.len() == 1 && reasons[0].as_str() == "end_turn"
+ if reasons.len() == 1 && reasons[0].as_str() == "stop"
));
Ok(())
}
diff --git a/crates/protocol/src/stream.rs b/crates/protocol/src/stream.rs
index 37bf123c8..c59a66f77 100644
--- a/crates/protocol/src/stream.rs
+++ b/crates/protocol/src/stream.rs
@@ -297,7 +297,7 @@ fn push_checked_chunk(
/// One provider-neutral streaming response chunk.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum LlmResponseChunk {
- /// Starts a response message.
+ /// Starts a response message, or fills in identity first supplied by a later frame.
MessageStart {
/// Provider response identifier.
id: Option<String>,
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index e87e21728..86fdf124d 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -67,14 +67,17 @@ fn decode_openai_chat_stream(
}
let mut out = Vec::new();
- if !state.saw_message_start {
+ let mut identity_changed = false;
+ if state.model.is_none() {
+ state.model = string_field(object, "model");
+ identity_changed |= state.model.is_some();
+ }
+ if state.message_id.is_none() {
+ state.message_id = string_field(object, "id");
+ identity_changed |= state.message_id.is_some();
+ }
+ if !state.saw_message_start || identity_changed {
state.saw_message_start = true;
- if let Some(model) = string_field(object, "model") {
- state.model = Some(model);
- }
- if let Some(id) = string_field(object, "id") {
- state.message_id = Some(id);
- }
out.push(LlmResponseChunk::MessageStart {
id: state.message_id.clone(),
model: state.model.clone(),
diff --git a/crates/switchyard-translation/src/codecs/responses/stream.rs b/crates/switchyard-translation/src/codecs/responses/stream.rs
index 399b0c3e2..7aa6df092 100644
--- a/crates/switchyard-translation/src/codecs/responses/stream.rs
+++ b/crates/switchyard-translation/src/codecs/responses/stream.rs
@@ -732,6 +732,10 @@ fn ensure_responses_created(state: &mut StreamTranslationState) -> Vec<Value> {
if state.response_created {
return Vec::new();
}
+ // A provider ID learned later must not change an already published response or item ID.
+ if state.target_message_id.is_none() {
+ state.target_message_id = Some(responses_id(state));
+ }
state.response_created = true;
vec![json!({
"type": "response.created",