Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: improve OpenRouter protocol interoperability (#237)
nachiketb-nvidia3087e1e3d494
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex 4627f964d..542922820 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -45,6 +45,7 @@ const RESERVED_HEADERS: &[&str] = &["anthropic-beta","anthropic-version","content-type",+ "accept-encoding",];const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250);@@ -1547,6 +1548,7 @@ mod tests {let mut headers = BTreeMap::new();headers.insert("x-request-id".to_string(), "abc".to_string());headers.insert("authorization".to_string(), "Bearer client-key".to_string());+ headers.insert("accept-encoding".to_string(), "gzip, br".to_string());let request = Request {llm_request: LlmRequest {model: Some("gpt".to_string()),@@ -1572,6 +1574,12 @@ mod tests {client.call_rewrite_model(Context::default(), request, None).await?;+ let received = server+ .received_requests()+ .await+ .ok_or("request recording should be enabled")?;+ let received = received.first().ok_or("expected one upstream request")?;+ assert!(!received.headers.contains_key("accept-encoding"));Ok(())}diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rsindex 28e3f80e6..51dc96a3d 100644--- a/crates/switchyard-server/src/stats/accumulator.rs+++ b/crates/switchyard-server/src/stats/accumulator.rs@@ -66,6 +66,19 @@ impl StatsAccumulator {}}+ /// Records a stream failure after its routed call was already counted.+ pub(crate) fn record_stream_error(&self, model: impl Into<String>, tier: Option<&str>) {+ let mut inner = self.lock();+ inner.total_errors = inner.total_errors.saturating_add(1);+ let model = model.into();+ let stats = inner.model_stats_mut(model.clone());+ stats.errors = stats.errors.saturating_add(1);+ if let Some(tier) = normalized_tier(tier) {+ stats.tiers.insert(tier.to_string());+ inner.tier_stats_mut(tier, &model);+ }+ }+/// Records usage and terminal latency after a successful routed call.pub(crate) fn record_usage(&self,diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rsindex f6d050e1e..56bc44745 100644--- a/crates/switchyard-server/src/usage_metrics.rs+++ b/crates/switchyard-server/src/usage_metrics.rs@@ -28,7 +28,10 @@ pub(crate) fn observe(metadata,} = response;let model = model.to_string();- let tier = tier.map(str::to_string);+ let tier = tier+ .map(str::trim)+ .filter(|tier| !tier.is_empty())+ .map(str::to_string);let llm_response = match llm_response {LlmResponse::Agg(agg) => {@@ -60,6 +63,9 @@ pub(crate) fn observe(if let Ok(LlmResponseChunk::Usage(usage)) = &item {latest_usage = Some(usage.clone());}+ if failed {+ record_stream_error(&stats, &model, tier.as_deref());+ }yield item;if failed {return;@@ -88,6 +94,15 @@ pub(crate) fn observe(}}+// Records a terminal stream failure after the routed call was already counted.+fn record_stream_error(stats: &StatsAccumulator, model: &str, tier: Option<&str>) {+ stats.record_stream_error(model, tier);+ global::meter("switchyard")+ .u64_counter("switchyard.errors")+ .build()+ .add(1, &attributes(model, tier));+}+pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {let cached_tokens = usage.cached_input_tokens().unwrap_or(0);let cache_creation_tokens = usage.cache_creation_input_tokens().unwrap_or(0);diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex a2fcba76c..b9aa45854 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -1130,7 +1130,8 @@ async fn streaming_success_records_only_final_usage_and_one_latency() -> TestRes}#[tokio::test]-async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {+// A terminal stream failure records errors without usage or terminal latency.+async fn streaming_error_records_error_without_usage_or_latency() -> TestResult {const MODEL: &str = "model/stream-error";let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;let before = send(&app, "GET", "/metrics", None).await?;@@ -1170,10 +1171,21 @@ async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {"{name} changed after a failed stream");}+ assert_eq!(+ metric_delta(+ before,+ after,+ "switchyard_errors_total",+ &[("model", MODEL)]+ ),+ Some(1.0)+ );let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;assert_eq!(stats["total_requests"], 1);+ assert_eq!(stats["total_errors"], 1);assert_eq!(stats["total_tokens"], empty_token_totals());assert_eq!(stats["models"][MODEL]["calls"], 1);+ assert_eq!(stats["models"][MODEL]["errors"], 1);assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 0);assert_eq!(stats["routing_overhead"]["count"], 1);Ok(())diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rsindex d7b0e3b47..b11922e07 100644--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs@@ -202,7 +202,11 @@ fn encode_openai_chat_stream(LlmResponseChunk::Usage(usage) => {state.usage = usage;state.saw_backend_usage = true;- Vec::new()+ if state.finished {+ vec![openai_usage_chunk(state)]+ } else {+ Vec::new()+ }}LlmResponseChunk::MessageStop { reason } => {if state.finished {@@ -213,7 +217,7 @@ fn encode_openai_chat_stream(state,json!({}),Some(openai_finish_reason(reason.as_deref())),- Some(openai_usage_value(state)),+ state.saw_backend_usage.then(|| openai_usage_value(state)),)]}LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {@@ -232,7 +236,7 @@ fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec<Value> {state,json!({}),Some(openai_finish_reason(state.stop_reason.as_deref())),- Some(openai_usage_value(state)),+ state.saw_backend_usage.then(|| openai_usage_value(state)),)]}@@ -298,6 +302,13 @@ fn openai_stream_chunk(payload}+// Builds the usage-only chunk OpenAI emits after the terminal choices chunk.+fn openai_usage_chunk(state: &StreamTranslationState) -> Value {+ let mut payload = openai_stream_chunk(state, json!({}), None, Some(openai_usage_value(state)));+ payload["choices"] = json!([]);+ payload+}+// Builds an OpenAI Chat tool-call delta chunk.fn openai_tool_call_chunk(state: &StreamTranslationState,diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rsindex 508d225fc..dbb047514 100644--- a/crates/switchyard-translation/src/helpers.rs+++ b/crates/switchyard-translation/src/helpers.rs@@ -153,14 +153,18 @@ wherelet line = line.map_err(llm_client_error_from_io)?;// A blank line (allowing a bare CR for CRLF streams) ends the frame.if line.trim_end().is_empty() {- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?- {- for event in codec.decode_event(&mut state, &value) {- yield event;+ let parsed = sse::parse_json_sse_frame(&frame, marker)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ frame.clear();+ match parsed {+ sse::SseFrame::Empty => {}+ sse::SseFrame::Done => break,+ sse::SseFrame::Data(value) => {+ for event in codec.decode_event(&mut state, &value) {+ yield event;+ }}}- frame.clear();} else {frame.push_str(&line);frame.push('\n');@@ -171,9 +175,9 @@ where// complete frame instead of losing its last chunk.#[allow(clippy::collapsible_if)]if !frame.trim_end().is_empty() {- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?- {+ let parsed = sse::parse_json_sse_frame(&frame, marker)+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;+ if let sse::SseFrame::Data(value) = parsed {for event in codec.decode_event(&mut state, &value) {yield event;}@@ -384,7 +388,8 @@ mod tests {fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), LlmClientError> {let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\- data: [DONE]\n\n"+ data: [DONE]\n\n\+ data: {\"choices\":[{\"delta\":{\"content\":\" ignored\"}}]}\n\n".to_vec();let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;diff --git a/crates/switchyard-translation/src/sse.rs b/crates/switchyard-translation/src/sse.rsindex 06864bafb..dce5a3611 100644--- a/crates/switchyard-translation/src/sse.rs+++ b/crates/switchyard-translation/src/sse.rs@@ -14,20 +14,25 @@ use crate::WireFormat;/// Boxed, thread-safe error carried by a streamed item.pub(crate) type BoxError = Box<dyn std::error::Error + Send + Sync>;-/// The terminal SSE marker for `format`, if any. OpenAI Chat and Responses use-/// `[DONE]`; Anthropic ends on its `message_stop` event with no marker.+/// Parsed contents of one SSE frame.+pub(crate) enum SseFrame {+ Empty,+ Done,+ Data(Value),+}++/// The optional terminal SSE marker accepted for all supported wire formats.+/// Native Anthropic streams can end at `message_stop`, while compatible+/// providers may additionally send `[DONE]`.#[inline]-pub(crate) fn done_marker(format: WireFormat) -> Option<&'static str> {- match format {- WireFormat::OpenAiChat | WireFormat::OpenAiResponses => Some("[DONE]"),- WireFormat::AnthropicMessages => None,- }+pub(crate) fn done_marker(_format: WireFormat) -> Option<&'static str> {+ Some("[DONE]")}pub(crate) fn parse_json_sse_frame(frame: &str,done_marker: Option<&str>,-) -> Result<Option<Value>, BoxError> {+) -> Result<SseFrame, BoxError> {let data = frame.lines().filter(|line| !line.is_empty() && !line.starts_with(':'))@@ -40,12 +45,14 @@ pub(crate) fn parse_json_sse_frame(});let data = data.trim_end();- // No data payload, or the terminal marker: nothing to decode.- if data.is_empty() || done_marker.is_some_and(|marker| data == marker) {- return Ok(None);+ if data.is_empty() {+ return Ok(SseFrame::Empty);+ }+ if done_marker.is_some_and(|marker| data == marker) {+ return Ok(SseFrame::Done);}let value = serde_json::from_str::<Value>(data)?;- Ok(Some(value))+ Ok(SseFrame::Data(value))}#[cfg(test)]@@ -58,8 +65,9 @@ mod tests {#[test]fn parses_a_data_line_as_json() -> Result<(), BoxError> {- let value =- parse_json_sse_frame("data: {\"text\":\"hi\"}\n", DONE)?.ok_or("expected a payload")?;+ let SseFrame::Data(value) = parse_json_sse_frame("data: {\"text\":\"hi\"}\n", DONE)? else {+ return Err("expected a payload".into());+ };assert_eq!(value, json!({"text": "hi"}));Ok(())}@@ -68,23 +76,31 @@ mod tests {fn ignores_comment_and_non_data_fields() -> Result<(), BoxError> {// Only `data:` fields contribute; comments (`:`) and other fields (`event:`) are dropped.let frame = ": keep-alive\nevent: message\ndata: {\"n\":1}\n";- let value = parse_json_sse_frame(frame, DONE)?.ok_or("expected a payload")?;+ let SseFrame::Data(value) = parse_json_sse_frame(frame, DONE)? else {+ return Err("expected a payload".into());+ };assert_eq!(value, json!({"n": 1}));Ok(())}#[test]fn done_marker_yields_no_payload() -> Result<(), BoxError> {- assert!(parse_json_sse_frame("data: [DONE]\n", DONE)?.is_none());+ assert!(matches!(+ parse_json_sse_frame("data: [DONE]\n", DONE)?,+ SseFrame::Done+ ));Ok(())}#[test]fn frame_without_data_yields_no_payload() -> Result<(), BoxError> {// A comment-only frame carries no data payload.- assert!(parse_json_sse_frame(": keep-alive\n", DONE)?.is_none());+ assert!(matches!(+ parse_json_sse_frame(": keep-alive\n", DONE)?,+ SseFrame::Empty+ ));// An empty frame likewise decodes to nothing.- assert!(parse_json_sse_frame("", DONE)?.is_none());+ assert!(matches!(parse_json_sse_frame("", DONE)?, SseFrame::Empty));Ok(())}@@ -106,7 +122,7 @@ mod tests {}#[test]- fn anthropic_has_no_done_marker() {- assert_eq!(done_marker(WireFormat::AnthropicMessages), None);+ fn anthropic_accepts_optional_done_marker() {+ assert_eq!(done_marker(WireFormat::AnthropicMessages), Some("[DONE]"));}}diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rsindex ec3908936..38a6f628c 100644--- a/crates/switchyard-translation/tests/stream_translation.rs+++ b/crates/switchyard-translation/tests/stream_translation.rs@@ -217,7 +217,45 @@ fn openai_chat_finish_synthesizes_terminal_chunk_after_incomplete_source() -> Te};assert_eq!(terminal["choices"][0]["delta"], json!({}));assert_eq!(terminal["choices"][0]["finish_reason"], "stop");- assert_eq!(terminal["usage"]["total_tokens"], 0);+ assert!(terminal.get("usage").is_none());+ Ok(())+}++// Verifies provider usage arriving after finish remains visible to OpenAI clients.+#[test]+fn openai_chat_emits_usage_arriving_after_stop() -> TestResult {+ let engine = TranslationEngine::default();+ let mut state = StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiChat);+ let stop = json!({+ "id": "chatcmpl-test",+ "object": "chat.completion.chunk",+ "model": "gpt-4o",+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]+ });+ let events = engine.translate_event(+ &mut state,+ WireFormat::OpenAiChat,+ WireFormat::OpenAiChat,+ &stop,+ )?;+ assert!(events[0].get("usage").is_none());++ let usage = json!({+ "id": "chatcmpl-test",+ "object": "chat.completion.chunk",+ "model": "gpt-4o",+ "choices": [],+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}+ });+ let events = engine.translate_event(+ &mut state,+ WireFormat::OpenAiChat,+ WireFormat::OpenAiChat,+ &usage,+ )?;+ assert_eq!(events.len(), 1);+ assert_eq!(events[0]["choices"], json!([]));+ assert_eq!(events[0]["usage"]["total_tokens"], 15);Ok(())}