Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: improve OpenRouter protocol interoperability (#237)

nachiketb-nvidia3087e1e3d494
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index 4627f964d..542922820 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -45,6 +45,7 @@ const RESERVED_HEADERS: &[&str] = &[
"anthropic-beta",
"anthropic-version",
"content-type",
+ "accept-encoding",
];
const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(250);
@@ -1547,6 +1548,7 @@ mod tests {
let mut headers = BTreeMap::new();
headers.insert("x-request-id".to_string(), "abc".to_string());
headers.insert("authorization".to_string(), "Bearer client-key".to_string());
+ headers.insert("accept-encoding".to_string(), "gzip, br".to_string());
let request = Request {
llm_request: LlmRequest {
model: Some("gpt".to_string()),
@@ -1572,6 +1574,12 @@ mod tests {
client
.call_rewrite_model(Context::default(), request, None)
.await?;
+ let received = server
+ .received_requests()
+ .await
+ .ok_or("request recording should be enabled")?;
+ let received = received.first().ok_or("expected one upstream request")?;
+ assert!(!received.headers.contains_key("accept-encoding"));
Ok(())
}
diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs
index 28e3f80e6..51dc96a3d 100644
--- a/crates/switchyard-server/src/stats/accumulator.rs
+++ b/crates/switchyard-server/src/stats/accumulator.rs
@@ -66,6 +66,19 @@ impl StatsAccumulator {
}
}
+ /// Records a stream failure after its routed call was already counted.
+ pub(crate) fn record_stream_error(&self, model: impl Into<String>, tier: Option<&str>) {
+ let mut inner = self.lock();
+ inner.total_errors = inner.total_errors.saturating_add(1);
+ let model = model.into();
+ let stats = inner.model_stats_mut(model.clone());
+ stats.errors = stats.errors.saturating_add(1);
+ if let Some(tier) = normalized_tier(tier) {
+ stats.tiers.insert(tier.to_string());
+ inner.tier_stats_mut(tier, &model);
+ }
+ }
+
/// Records usage and terminal latency after a successful routed call.
pub(crate) fn record_usage(
&self,
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
index f6d050e1e..56bc44745 100644
--- a/crates/switchyard-server/src/usage_metrics.rs
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -28,7 +28,10 @@ pub(crate) fn observe(
metadata,
} = response;
let model = model.to_string();
- let tier = tier.map(str::to_string);
+ let tier = tier
+ .map(str::trim)
+ .filter(|tier| !tier.is_empty())
+ .map(str::to_string);
let llm_response = match llm_response {
LlmResponse::Agg(agg) => {
@@ -60,6 +63,9 @@ pub(crate) fn observe(
if let Ok(LlmResponseChunk::Usage(usage)) = &item {
latest_usage = Some(usage.clone());
}
+ if failed {
+ record_stream_error(&stats, &model, tier.as_deref());
+ }
yield item;
if failed {
return;
@@ -88,6 +94,15 @@ pub(crate) fn observe(
}
}
+// Records a terminal stream failure after the routed call was already counted.
+fn record_stream_error(stats: &StatsAccumulator, model: &str, tier: Option<&str>) {
+ stats.record_stream_error(model, tier);
+ global::meter("switchyard")
+ .u64_counter("switchyard.errors")
+ .build()
+ .add(1, &attributes(model, tier));
+}
+
pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {
let cached_tokens = usage.cached_input_tokens().unwrap_or(0);
let cache_creation_tokens = usage.cache_creation_input_tokens().unwrap_or(0);
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index a2fcba76c..b9aa45854 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -1130,7 +1130,8 @@ async fn streaming_success_records_only_final_usage_and_one_latency() -> TestRes
}
#[tokio::test]
-async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {
+// A terminal stream failure records errors without usage or terminal latency.
+async fn streaming_error_records_error_without_usage_or_latency() -> TestResult {
const MODEL: &str = "model/stream-error";
let (_upstream, app) = test_app(&[(ROUTE_MODEL, &[MODEL])]).await?;
let before = send(&app, "GET", "/metrics", None).await?;
@@ -1170,10 +1171,21 @@ async fn streaming_error_records_neither_usage_nor_latency() -> TestResult {
"{name} changed after a failed stream"
);
}
+ assert_eq!(
+ metric_delta(
+ before,
+ after,
+ "switchyard_errors_total",
+ &[("model", MODEL)]
+ ),
+ Some(1.0)
+ );
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 1);
+ assert_eq!(stats["total_errors"], 1);
assert_eq!(stats["total_tokens"], empty_token_totals());
assert_eq!(stats["models"][MODEL]["calls"], 1);
+ assert_eq!(stats["models"][MODEL]["errors"], 1);
assert_eq!(stats["models"][MODEL]["total_latency"]["count"], 0);
assert_eq!(stats["routing_overhead"]["count"], 1);
Ok(())
diff --git a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
index d7b0e3b47..b11922e07 100644
--- a/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
+++ b/crates/switchyard-translation/src/codecs/openai_chat/stream.rs
@@ -202,7 +202,11 @@ fn encode_openai_chat_stream(
LlmResponseChunk::Usage(usage) => {
state.usage = usage;
state.saw_backend_usage = true;
- Vec::new()
+ if state.finished {
+ vec![openai_usage_chunk(state)]
+ } else {
+ Vec::new()
+ }
}
LlmResponseChunk::MessageStop { reason } => {
if state.finished {
@@ -213,7 +217,7 @@ fn encode_openai_chat_stream(
state,
json!({}),
Some(openai_finish_reason(reason.as_deref())),
- Some(openai_usage_value(state)),
+ state.saw_backend_usage.then(|| openai_usage_value(state)),
)]
}
LlmResponseChunk::DecodeError { message } | LlmResponseChunk::StreamError { message } => {
@@ -232,7 +236,7 @@ fn finish_openai_chat_stream(state: &mut StreamTranslationState) -> Vec<Value> {
state,
json!({}),
Some(openai_finish_reason(state.stop_reason.as_deref())),
- Some(openai_usage_value(state)),
+ state.saw_backend_usage.then(|| openai_usage_value(state)),
)]
}
@@ -298,6 +302,13 @@ fn openai_stream_chunk(
payload
}
+// Builds the usage-only chunk OpenAI emits after the terminal choices chunk.
+fn openai_usage_chunk(state: &StreamTranslationState) -> Value {
+ let mut payload = openai_stream_chunk(state, json!({}), None, Some(openai_usage_value(state)));
+ payload["choices"] = json!([]);
+ payload
+}
+
// Builds an OpenAI Chat tool-call delta chunk.
fn openai_tool_call_chunk(
state: &StreamTranslationState,
diff --git a/crates/switchyard-translation/src/helpers.rs b/crates/switchyard-translation/src/helpers.rs
index 508d225fc..dbb047514 100644
--- a/crates/switchyard-translation/src/helpers.rs
+++ b/crates/switchyard-translation/src/helpers.rs
@@ -153,14 +153,18 @@ where
let line = line.map_err(llm_client_error_from_io)?;
// A blank line (allowing a bare CR for CRLF streams) ends the frame.
if line.trim_end().is_empty() {
- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?
- {
- for event in codec.decode_event(&mut state, &value) {
- yield event;
+ let parsed = sse::parse_json_sse_frame(&frame, marker)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ frame.clear();
+ match parsed {
+ sse::SseFrame::Empty => {}
+ sse::SseFrame::Done => break,
+ sse::SseFrame::Data(value) => {
+ for event in codec.decode_event(&mut state, &value) {
+ yield event;
+ }
}
}
- frame.clear();
} else {
frame.push_str(&line);
frame.push('\n');
@@ -171,9 +175,9 @@ where
// complete frame instead of losing its last chunk.
#[allow(clippy::collapsible_if)]
if !frame.trim_end().is_empty() {
- if let Some(value) = sse::parse_json_sse_frame(&frame, marker)
- .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?
- {
+ let parsed = sse::parse_json_sse_frame(&frame, marker)
+ .map_err(|error| LlmClientError::ResponseTranslation(error.to_string()))?;
+ if let sse::SseFrame::Data(value) = parsed {
for event in codec.decode_event(&mut state, &value) {
yield event;
}
@@ -384,7 +388,8 @@ mod tests {
fn decode_stream_parses_sse_bytes_into_ir_chunks() -> Result<(), LlmClientError> {
let sse = b"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}\n\n\
data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}\n\n\
- data: [DONE]\n\n"
+ data: [DONE]\n\n\
+ data: {\"choices\":[{\"delta\":{\"content\":\" ignored\"}}]}\n\n"
.to_vec();
let bytes = stream::once(async move { Ok::<Vec<u8>, LlmClientError>(sse) });
let chunks = decode_all(bytes, WireFormat::OpenAiChat)?;
diff --git a/crates/switchyard-translation/src/sse.rs b/crates/switchyard-translation/src/sse.rs
index 06864bafb..dce5a3611 100644
--- a/crates/switchyard-translation/src/sse.rs
+++ b/crates/switchyard-translation/src/sse.rs
@@ -14,20 +14,25 @@ use crate::WireFormat;
/// Boxed, thread-safe error carried by a streamed item.
pub(crate) type BoxError = Box<dyn std::error::Error + Send + Sync>;
-/// The terminal SSE marker for `format`, if any. OpenAI Chat and Responses use
-/// `[DONE]`; Anthropic ends on its `message_stop` event with no marker.
+/// Parsed contents of one SSE frame.
+pub(crate) enum SseFrame {
+ Empty,
+ Done,
+ Data(Value),
+}
+
+/// The optional terminal SSE marker accepted for all supported wire formats.
+/// Native Anthropic streams can end at `message_stop`, while compatible
+/// providers may additionally send `[DONE]`.
#[inline]
-pub(crate) fn done_marker(format: WireFormat) -> Option<&'static str> {
- match format {
- WireFormat::OpenAiChat | WireFormat::OpenAiResponses => Some("[DONE]"),
- WireFormat::AnthropicMessages => None,
- }
+pub(crate) fn done_marker(_format: WireFormat) -> Option<&'static str> {
+ Some("[DONE]")
}
pub(crate) fn parse_json_sse_frame(
frame: &str,
done_marker: Option<&str>,
-) -> Result<Option<Value>, BoxError> {
+) -> Result<SseFrame, BoxError> {
let data = frame
.lines()
.filter(|line| !line.is_empty() && !line.starts_with(':'))
@@ -40,12 +45,14 @@ pub(crate) fn parse_json_sse_frame(
});
let data = data.trim_end();
- // No data payload, or the terminal marker: nothing to decode.
- if data.is_empty() || done_marker.is_some_and(|marker| data == marker) {
- return Ok(None);
+ if data.is_empty() {
+ return Ok(SseFrame::Empty);
+ }
+ if done_marker.is_some_and(|marker| data == marker) {
+ return Ok(SseFrame::Done);
}
let value = serde_json::from_str::<Value>(data)?;
- Ok(Some(value))
+ Ok(SseFrame::Data(value))
}
#[cfg(test)]
@@ -58,8 +65,9 @@ mod tests {
#[test]
fn parses_a_data_line_as_json() -> Result<(), BoxError> {
- let value =
- parse_json_sse_frame("data: {\"text\":\"hi\"}\n", DONE)?.ok_or("expected a payload")?;
+ let SseFrame::Data(value) = parse_json_sse_frame("data: {\"text\":\"hi\"}\n", DONE)? else {
+ return Err("expected a payload".into());
+ };
assert_eq!(value, json!({"text": "hi"}));
Ok(())
}
@@ -68,23 +76,31 @@ mod tests {
fn ignores_comment_and_non_data_fields() -> Result<(), BoxError> {
// Only `data:` fields contribute; comments (`:`) and other fields (`event:`) are dropped.
let frame = ": keep-alive\nevent: message\ndata: {\"n\":1}\n";
- let value = parse_json_sse_frame(frame, DONE)?.ok_or("expected a payload")?;
+ let SseFrame::Data(value) = parse_json_sse_frame(frame, DONE)? else {
+ return Err("expected a payload".into());
+ };
assert_eq!(value, json!({"n": 1}));
Ok(())
}
#[test]
fn done_marker_yields_no_payload() -> Result<(), BoxError> {
- assert!(parse_json_sse_frame("data: [DONE]\n", DONE)?.is_none());
+ assert!(matches!(
+ parse_json_sse_frame("data: [DONE]\n", DONE)?,
+ SseFrame::Done
+ ));
Ok(())
}
#[test]
fn frame_without_data_yields_no_payload() -> Result<(), BoxError> {
// A comment-only frame carries no data payload.
- assert!(parse_json_sse_frame(": keep-alive\n", DONE)?.is_none());
+ assert!(matches!(
+ parse_json_sse_frame(": keep-alive\n", DONE)?,
+ SseFrame::Empty
+ ));
// An empty frame likewise decodes to nothing.
- assert!(parse_json_sse_frame("", DONE)?.is_none());
+ assert!(matches!(parse_json_sse_frame("", DONE)?, SseFrame::Empty));
Ok(())
}
@@ -106,7 +122,7 @@ mod tests {
}
#[test]
- fn anthropic_has_no_done_marker() {
- assert_eq!(done_marker(WireFormat::AnthropicMessages), None);
+ fn anthropic_accepts_optional_done_marker() {
+ assert_eq!(done_marker(WireFormat::AnthropicMessages), Some("[DONE]"));
}
}
diff --git a/crates/switchyard-translation/tests/stream_translation.rs b/crates/switchyard-translation/tests/stream_translation.rs
index ec3908936..38a6f628c 100644
--- a/crates/switchyard-translation/tests/stream_translation.rs
+++ b/crates/switchyard-translation/tests/stream_translation.rs
@@ -217,7 +217,45 @@ fn openai_chat_finish_synthesizes_terminal_chunk_after_incomplete_source() -> Te
};
assert_eq!(terminal["choices"][0]["delta"], json!({}));
assert_eq!(terminal["choices"][0]["finish_reason"], "stop");
- assert_eq!(terminal["usage"]["total_tokens"], 0);
+ assert!(terminal.get("usage").is_none());
+ Ok(())
+}
+
+// Verifies provider usage arriving after finish remains visible to OpenAI clients.
+#[test]
+fn openai_chat_emits_usage_arriving_after_stop() -> TestResult {
+ let engine = TranslationEngine::default();
+ let mut state = StreamTranslationState::new(WireFormat::OpenAiChat, WireFormat::OpenAiChat);
+ let stop = json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]
+ });
+ let events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiChat,
+ &stop,
+ )?;
+ assert!(events[0].get("usage").is_none());
+
+ let usage = json!({
+ "id": "chatcmpl-test",
+ "object": "chat.completion.chunk",
+ "model": "gpt-4o",
+ "choices": [],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
+ });
+ let events = engine.translate_event(
+ &mut state,
+ WireFormat::OpenAiChat,
+ WireFormat::OpenAiChat,
+ &usage,
+ )?;
+ assert_eq!(events.len(), 1);
+ assert_eq!(events[0]["choices"], json!([]));
+ assert_eq!(events[0]["usage"]["total_tokens"], 15);
Ok(())
}