Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: Calculate stats for streaming responses (#64)

Graham Kingefbf05171a08
diff --git a/Cargo.lock b/Cargo.lock
index e73214a56..0beaae069 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1585,8 +1585,10 @@ dependencies = [
name = "switchyard-components-v2"
version = "0.1.0"
dependencies = [
+ "async-stream",
"async-trait",
"futures-core",
+ "futures-util",
"parking_lot",
"rand 0.8.6",
"reqwest",
diff --git a/crates/switchyard-components-v2/Cargo.toml b/crates/switchyard-components-v2/Cargo.toml
index d257a065d..f5b78dd7e 100644
--- a/crates/switchyard-components-v2/Cargo.toml
+++ b/crates/switchyard-components-v2/Cargo.toml
@@ -12,7 +12,9 @@ repository.workspace = true
rust-version.workspace = true
[dependencies]
+async-stream = "0.3"
async-trait = "0.1"
+futures-util = "0.3"
parking_lot = "0.12"
rand = "0.8"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] }
diff --git a/crates/switchyard-components-v2/src/lib.rs b/crates/switchyard-components-v2/src/lib.rs
index 201b04a5f..bfeaa30ae 100644
--- a/crates/switchyard-components-v2/src/lib.rs
+++ b/crates/switchyard-components-v2/src/lib.rs
@@ -13,6 +13,7 @@ mod config;
mod profile;
pub mod profiles;
mod stats;
+mod stats_recording;
pub use config::{
parse_profile_config_path, parse_profile_config_str, parse_profile_config_str_with_env_lookup,
diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing.rs b/crates/switchyard-components-v2/src/profiles/llm_routing.rs
index 66cd572eb..f44ca5175 100644
--- a/crates/switchyard-components-v2/src/profiles/llm_routing.rs
+++ b/crates/switchyard-components-v2/src/profiles/llm_routing.rs
@@ -17,6 +17,7 @@ use switchyard_core::{
use crate::backend::{native_target_backend, TargetBackend};
use crate::profile_stats_accumulator;
+use crate::stats_recording::record_usage_or_wrap_stream;
use crate::{
profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
RoutingMetadata,
@@ -421,25 +422,23 @@ impl LlmRoutingProfile {
fn record_success(
&self,
decision: &LlmRoutingDecision,
- response: &ChatResponse,
- total_latency_ms: f64,
+ response: ChatResponse,
+ profile_started_at: Instant,
backend_latency_ms: f64,
- ) -> Result<()> {
+ ) -> Result<ChatResponse> {
self.stats.record_success(
decision.selected_model.as_str(),
Some(backend_latency_ms),
Some(decision.tier.as_str()),
)?;
- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- self.stats.record_usage_after_success_attribution(
+ record_usage_or_wrap_stream(
+ &self.stats,
decision.selected_model.as_str(),
- usage,
- Some(total_latency_ms),
- Some(routing_overhead_ms),
Some(decision.tier.as_str()),
- )?;
- Ok(())
+ profile_started_at,
+ backend_latency_ms,
+ response,
+ )
}
fn record_error(&self, decision: &LlmRoutingDecision) -> Result<()> {
@@ -489,11 +488,10 @@ impl Profile for LlmRoutingProfile {
let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await;
match first_result {
Ok(response) => {
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- self.record_success(
+ let response = self.record_success(
&processed.decision,
- &response,
- total_latency_ms,
+ response,
+ profile_started_at,
first_backend_latency_ms,
)?;
let response = self.rprocess(&processed, response).await?;
@@ -507,11 +505,10 @@ impl Profile for LlmRoutingProfile {
let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await;
match retry_result {
Ok(response) => {
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- self.record_success(
+ let response = self.record_success(
&retry.decision,
- &response,
- total_latency_ms,
+ response,
+ profile_started_at,
retry_backend_latency_ms,
)?;
let response = self.rprocess(&retry, response).await?;
diff --git a/crates/switchyard-components-v2/src/profiles/passthrough.rs b/crates/switchyard-components-v2/src/profiles/passthrough.rs
index 2becb79f7..8d9edb892 100644
--- a/crates/switchyard-components-v2/src/profiles/passthrough.rs
+++ b/crates/switchyard-components-v2/src/profiles/passthrough.rs
@@ -6,12 +6,12 @@
use std::time::Instant;
use async_trait::async_trait;
-use switchyard_components::stats::usage_from_body;
use switchyard_components::StatsAccumulator;
use switchyard_core::{ChatResponse, LlmTarget, Result};
use crate::backend::{native_target_backend, TargetBackend};
use crate::profile_stats_accumulator;
+use crate::stats_recording::record_usage_or_wrap_stream;
use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse};
/// Config for the flatter passthrough profile.
@@ -85,15 +85,13 @@ impl Profile for PassthroughProfile {
let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;
self.stats
.record_success(target_model.to_string(), Some(backend_latency_ms), None)?;
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- self.stats.record_usage_after_success_attribution(
- target_model.to_string(),
- usage,
- Some(total_latency_ms),
- Some(routing_overhead_ms),
+ let response = record_usage_or_wrap_stream(
+ &self.stats,
+ target_model.as_str(),
None,
+ profile_started_at,
+ backend_latency_ms,
+ response,
)?;
let response = self.rprocess(&processed, response).await?;
Ok(ProfileResponse::from(response))
diff --git a/crates/switchyard-components-v2/src/profiles/random_routing.rs b/crates/switchyard-components-v2/src/profiles/random_routing.rs
index f0fc47386..dea63c67f 100644
--- a/crates/switchyard-components-v2/src/profiles/random_routing.rs
+++ b/crates/switchyard-components-v2/src/profiles/random_routing.rs
@@ -9,12 +9,12 @@ use async_trait::async_trait;
use switchyard_components::request_processors::{
RandomRoutingDecision, RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier,
};
-use switchyard_components::stats::usage_from_body;
use switchyard_components::StatsAccumulator;
use switchyard_core::{ChatResponse, LlmTarget, Result, SwitchyardError};
use crate::backend::{native_target_backend, TargetBackend};
use crate::profile_stats_accumulator;
+use crate::stats_recording::record_usage_or_wrap_stream;
use crate::{
profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
RoutingMetadata,
@@ -167,15 +167,13 @@ impl Profile for RandomRoutingProfile {
Some(backend_latency_ms),
Some(decision.tier.as_str()),
)?;
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- self.stats.record_usage_after_success_attribution(
+ let response = record_usage_or_wrap_stream(
+ &self.stats,
decision.selected_model.as_str(),
- usage,
- Some(total_latency_ms),
- Some(routing_overhead_ms),
Some(decision.tier.as_str()),
+ profile_started_at,
+ backend_latency_ms,
+ response,
)?;
let response = self.rprocess(&processed, response).await?;
Ok(ProfileResponse::with_routing_metadata(
@@ -195,8 +193,11 @@ mod tests {
use std::sync::{Arc, Mutex};
use async_trait::async_trait;
+ use futures_util::StreamExt;
use serde_json::{json, Value};
- use switchyard_core::{BackendFormat, ChatRequest, LlmTargetId, ModelId, SwitchyardError};
+ use switchyard_core::{
+ BackendFormat, ChatRequest, LlmTargetId, ModelId, StreamEvent, SwitchyardError,
+ };
use crate::backend::{ProfileBackend, TargetBackend};
use crate::RequestMetadata;
@@ -214,6 +215,8 @@ mod tests {
calls: Arc<Mutex<Vec<ObservedCall>>>,
}
+ struct StreamingUsageBackend;
+
#[async_trait]
impl ProfileBackend for TestBackend {
async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {
@@ -235,6 +238,27 @@ mod tests {
}
}
+ #[async_trait]
+ impl ProfileBackend for StreamingUsageBackend {
+ async fn call(&self, _request: &ChatRequest) -> Result<ChatResponse> {
+ Ok(ChatResponse::OpenAiStream(Box::pin(
+ futures_util::stream::iter([
+ Ok(StreamEvent::Json(json!({
+ "choices": [{"delta": {"content": "ok"}}],
+ }))),
+ Ok(StreamEvent::Json(json!({
+ "choices": [],
+ "usage": {
+ "prompt_tokens": 3,
+ "completion_tokens": 4,
+ "total_tokens": 7,
+ },
+ }))),
+ ]),
+ )))
+ }
+ }
+
fn target(id: &str, model: &str) -> Result<LlmTarget> {
let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);
target.format = BackendFormat::OpenAi;
@@ -385,6 +409,60 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn run_records_streaming_usage_with_selected_random_tier() -> Result<()> {
+ let strong = target("strong", "frontier/model")?;
+ let weak = target("weak", "cheap/model")?;
+ let router_config = RandomRoutingProcessorConfig::new(strong.clone(), weak.clone())
+ .with_strong_probability(0.0)?
+ .with_rng_seed(Some(7));
+ let profile = RandomRoutingProfile {
+ router: RandomRoutingEngine::new(router_config)?,
+ strong_backend: TargetBackend::new(
+ strong,
+ Arc::new(TestBackend {
+ name: "strong-backend",
+ calls: Arc::new(Mutex::new(Vec::new())),
+ }),
+ ),
+ weak_backend: TargetBackend::new(weak, Arc::new(StreamingUsageBackend)),
+ stats: StatsAccumulator::new(),
+ };
+
+ let response = profile
+ .run(profile_input(ChatRequest::openai_chat(json!({
+ "model": "client/model",
+ "messages": [],
+ "stream": true,
+ }))))
+ .await?;
+ let ChatResponse::OpenAiStream(mut stream) = response.response else {
+ return Err(SwitchyardError::Other("expected streaming response".into()));
+ };
+ while let Some(event) = stream.next().await {
+ event?;
+ }
+
+ let snapshot = profile.stats.snapshot()?;
+ assert_eq!(snapshot.total_requests, 1);
+ assert_eq!(snapshot.total_tokens.prompt, 3);
+ assert_eq!(snapshot.total_tokens.completion, 4);
+ assert_eq!(snapshot.total_tokens.total, 7);
+ let model = snapshot
+ .models
+ .get("cheap/model")
+ .ok_or_else(|| SwitchyardError::Other("weak model stats should be present".into()))?;
+ assert_eq!(model.calls, 1);
+ assert_eq!(model.tier.as_deref(), Some("weak"));
+ assert_eq!(model.total_tokens, 7);
+ let tier = snapshot
+ .tiers
+ .get("weak")
+ .ok_or_else(|| SwitchyardError::Other("weak tier stats should be present".into()))?;
+ assert_eq!(tier.total_tokens, 7);
+ Ok(())
+ }
+
#[tokio::test]
async fn run_disambiguates_duplicate_target_models_without_context_state() -> Result<()> {
let (profile, calls) = profile(
diff --git a/crates/switchyard-components-v2/src/profiles/stage_router.rs b/crates/switchyard-components-v2/src/profiles/stage_router.rs
index 9a864308e..6aebe709e 100644
--- a/crates/switchyard-components-v2/src/profiles/stage_router.rs
+++ b/crates/switchyard-components-v2/src/profiles/stage_router.rs
@@ -17,6 +17,7 @@ use switchyard_core::{ChatRequest, ChatResponse, LlmTarget, LlmTargetId, Result,
use crate::backend::{native_target_backend, TargetBackend};
use crate::profile_stats_accumulator;
+use crate::stats_recording::record_usage_or_wrap_stream;
use crate::{
profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,
RoutingMetadata,
@@ -483,27 +484,26 @@ impl StageRouterProfile {
fn record_success(
&self,
decision: &StageRouterDecision,
- response: &ChatResponse,
- total_latency_ms: f64,
+ response: ChatResponse,
+ profile_started_at: Instant,
backend_latency_ms: f64,
- ) -> Result<()> {
- if let Some(stats) = self.stats_handle() {
- stats.record_success(
- decision.selected_model.as_str(),
- Some(backend_latency_ms),
- Some(decision.tier.as_str()),
- )?;
- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
- let usage = response.body().map(usage_from_body).unwrap_or_default();
- stats.record_usage_after_success_attribution(
- decision.selected_model.as_str(),
- usage,
- Some(total_latency_ms),
- Some(routing_overhead_ms),
- Some(decision.tier.as_str()),
- )?;
- }
- Ok(())
+ ) -> Result<ChatResponse> {
+ let Some(stats) = self.stats_handle() else {
+ return Ok(response);
+ };
+ stats.record_success(
+ decision.selected_model.as_str(),
+ Some(backend_latency_ms),
+ Some(decision.tier.as_str()),
+ )?;
+ record_usage_or_wrap_stream(
+ stats,
+ decision.selected_model.as_str(),
+ Some(decision.tier.as_str()),
+ profile_started_at,
+ backend_latency_ms,
+ response,
+ )
}
fn record_error(&self, decision: &StageRouterDecision) -> Result<()> {
@@ -561,11 +561,10 @@ impl Profile for StageRouterProfile {
let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await;
match first_result {
Ok(response) => {
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- self.record_success(
+ let response = self.record_success(
&processed.decision,
- &response,
- total_latency_ms,
+ response,
+ profile_started_at,
first_backend_latency_ms,
)?;
let response = self.rprocess(&processed, response).await?;
@@ -579,11 +578,10 @@ impl Profile for StageRouterProfile {
let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await;
match retry_result {
Ok(response) => {
- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
- self.record_success(
+ let response = self.record_success(
&retry.decision,
- &response,
- total_latency_ms,
+ response,
+ profile_started_at,
retry_backend_latency_ms,
)?;
let response = self.rprocess(&retry, response).await?;
diff --git a/crates/switchyard-components-v2/src/stats_recording.rs b/crates/switchyard-components-v2/src/stats_recording.rs
new file mode 100644
index 000000000..501d30b49
--- /dev/null
+++ b/crates/switchyard-components-v2/src/stats_recording.rs
@@ -0,0 +1,209 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Records provider token usage for buffered and streaming responses.
+//!
+//! Buffered responses carry usage in the body and can be recorded synchronously.
+//! Streaming responses only emit usage inside a later event, so the response
+//! stream is wrapped and usage is recorded when the first usage-bearing event
+//! flows through — no buffering, no added latency, and the client still sees
+//! every event.
+
+use std::time::Instant;
+
+use async_stream::try_stream;
+use futures_util::StreamExt;
+use switchyard_components::stats::{
+ openai_chat_usage_from_stream_event, openai_responses_usage_from_stream_event, usage_from_body,
+ AnthropicStreamUsage, StatsAccumulator, TokenUsage,
+};
+use switchyard_core::{BoxResponseStream, ChatResponse, Result};
+
+/// Records usage from a buffered response, or wraps a streaming response so
+/// usage is recorded when its usage-bearing event arrives.
+pub(crate) fn record_usage_or_wrap_stream(
+ stats: &StatsAccumulator,
+ model: &str,
+ tier: Option<&str>,
+ profile_started_at: Instant,
+ backend_latency_ms: f64,
+ response: ChatResponse,
+) -> Result<ChatResponse> {
+ // We report time-to-first-token as latency. Otherwise in streaming case we'd be including
+ // the client's processing time.
+ let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;
+ match response {
+ ChatResponse::OpenAiCompletion(_)
+ | ChatResponse::OpenAiResponsesCompletion(_)
+ | ChatResponse::AnthropicCompletion(_) => {
+ let usage = response.body().map(usage_from_body).unwrap_or_default();
+ record_usage(
+ stats,
+ model,
+ tier,
+ total_latency_ms,
+ backend_latency_ms,
+ usage,
+ )?;
+ Ok(response)
+ }
+ ChatResponse::OpenAiStream(stream) => Ok(ChatResponse::OpenAiStream(wrap_openai_chat(
+ stream,
+ stats.clone(),
+ model.to_string(),
+ tier.map(str::to_string),
+ total_latency_ms,
+ backend_latency_ms,
+ ))),
+ ChatResponse::OpenAiResponsesStream(stream) => {
+ Ok(ChatResponse::OpenAiResponsesStream(wrap_openai_responses(
+ stream,
+ stats.clone(),
+ model.to_string(),
+ tier.map(str::to_string),
+ total_latency_ms,
+ backend_latency_ms,
+ )))
+ }
+ ChatResponse::AnthropicStream(stream) => Ok(ChatResponse::AnthropicStream(wrap_anthropic(
+ stream,
+ stats.clone(),
+ model.to_string(),
+ tier.map(str::to_string),
+ total_latency_ms,
+ backend_latency_ms,
+ ))),
+ }
+}
+
+/// Wraps an OpenAI chat completions stream to record usage stats.
+///
+/// Usage is typically only present in the last JSON SSE chunk, but we only
+/// record the first instance to avoid double counting in case of middle-box weirdness.
+///
+/// Pass-through, no buffering or dropping.
+fn wrap_openai_chat(
+ mut stream: BoxResponseStream,
+ stats: StatsAccumulator,
+ model: String,
+ tier: Option<String>,
+ total_latency_ms: f64,
+ backend_latency_ms: f64,
+) -> BoxResponseStream {
+ Box::pin(try_stream! {
+ let mut committed = false;
+ while let Some(event) = stream.next().await {
+ let event = event?;
+ if !committed {
+ if let Some(usage) = openai_chat_usage_from_stream_event(&event) {
+ log_stream_record_result(
+ record_usage(
+ &stats,
+ &model,
+ tier.as_deref(),
+ total_latency_ms,
+ backend_latency_ms,
+ usage,
+ ),
+ &model,
+ );
+ committed = true;
+ }
+ }
+ yield event;
+ }
+ })
+}
+
+/// Wraps an OpenAI responses stream to record usage stats.
+fn wrap_openai_responses(
+ mut stream: BoxResponseStream,
+ stats: StatsAccumulator,
+ model: String,
+ tier: Option<String>,
+ total_latency_ms: f64,
+ backend_latency_ms: f64,
+) -> BoxResponseStream {
+ Box::pin(try_stream! {
+ let mut committed = false;
+ while let Some(event) = stream.next().await {
+ let event = event?;
+ if !committed {
+ if let Some(usage) = openai_responses_usage_from_stream_event(&event) {
+ log_stream_record_result(
+ record_usage(
+ &stats,
+ &model,
+ tier.as_deref(),
+ total_latency_ms,
+ backend_latency_ms,
+ usage,
+ ),
+ &model,
+ );
+ committed = true;
+ }
+ }
+ yield event;
+ }
+ })
+}
+
+/// Wraps an Anthropic messages stream to record usage stats.
+fn wrap_anthropic(
+ mut stream: BoxResponseStream,
+ stats: StatsAccumulator,
+ model: String,
+ tier: Option<String>,
+ total_latency_ms: f64,
+ backend_latency_ms: f64,
+) -> BoxResponseStream {
+ Box::pin(try_stream! {
+ let mut stream_usage = AnthropicStreamUsage::default();
+ while let Some(event) = stream.next().await {
+ let event = event?;
+ if let Some(usage) = stream_usage.observe(&event) {
+ log_stream_record_result(
+ record_usage(
+ &stats,
+ &model,
+ tier.as_deref(),
+ total_latency_ms,
+ backend_latency_ms,
+ usage,
+ ),
+ &model,
+ );
+ }
+ yield event;
+ }
+ })
+}
+
+fn record_usage(
+ stats: &StatsAccumulator,
+ model: &str,
+ tier: Option<&str>,
+ total_latency_ms: f64,
+ backend_latency_ms: f64,
+ usage: TokenUsage,
+) -> Result<()> {
+ let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);
+ stats.record_usage_after_success_attribution(
+ model.to_string(),
+ usage,
+ Some(total_latency_ms),
+ Some(routing_overhead_ms),
+ tier,
+ )
+}
+
+fn log_stream_record_result(result: Result<()>, model: &str) {
+ if let Err(error) = result {
+ tracing::warn!(
+ error = %error,
+ model = %model,
+ "failed to record stream usage",
+ );
+ }
+}