Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix: Calculate stats for streaming responses (#64)
Graham Kingefbf05171a08
diff --git a/Cargo.lock b/Cargo.lockindex e73214a56..0beaae069 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1585,8 +1585,10 @@ dependencies = [name = "switchyard-components-v2"version = "0.1.0"dependencies = [+ "async-stream","async-trait","futures-core",+ "futures-util","parking_lot","rand 0.8.6","reqwest",diff --git a/crates/switchyard-components-v2/Cargo.toml b/crates/switchyard-components-v2/Cargo.tomlindex d257a065d..f5b78dd7e 100644--- a/crates/switchyard-components-v2/Cargo.toml+++ b/crates/switchyard-components-v2/Cargo.toml@@ -12,7 +12,9 @@ repository.workspace = truerust-version.workspace = true[dependencies]+async-stream = "0.3"async-trait = "0.1"+futures-util = "0.3"parking_lot = "0.12"rand = "0.8"reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] }diff --git a/crates/switchyard-components-v2/src/lib.rs b/crates/switchyard-components-v2/src/lib.rsindex 201b04a5f..bfeaa30ae 100644--- a/crates/switchyard-components-v2/src/lib.rs+++ b/crates/switchyard-components-v2/src/lib.rs@@ -13,6 +13,7 @@ mod config;mod profile;pub mod profiles;mod stats;+mod stats_recording;pub use config::{parse_profile_config_path, parse_profile_config_str, parse_profile_config_str_with_env_lookup,diff --git a/crates/switchyard-components-v2/src/profiles/llm_routing.rs b/crates/switchyard-components-v2/src/profiles/llm_routing.rsindex 66cd572eb..f44ca5175 100644--- a/crates/switchyard-components-v2/src/profiles/llm_routing.rs+++ b/crates/switchyard-components-v2/src/profiles/llm_routing.rs@@ -17,6 +17,7 @@ use switchyard_core::{use crate::backend::{native_target_backend, TargetBackend};use crate::profile_stats_accumulator;+use crate::stats_recording::record_usage_or_wrap_stream;use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,RoutingMetadata,@@ -421,25 +422,23 @@ impl LlmRoutingProfile {fn record_success(&self,decision: &LlmRoutingDecision,- response: &ChatResponse,- total_latency_ms: f64,+ response: ChatResponse,+ profile_started_at: Instant,backend_latency_ms: f64,- ) -> Result<()> {+ ) -> Result<ChatResponse> {self.stats.record_success(decision.selected_model.as_str(),Some(backend_latency_ms),Some(decision.tier.as_str()),)?;- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);- let usage = response.body().map(usage_from_body).unwrap_or_default();- self.stats.record_usage_after_success_attribution(+ record_usage_or_wrap_stream(+ &self.stats,decision.selected_model.as_str(),- usage,- Some(total_latency_ms),- Some(routing_overhead_ms),Some(decision.tier.as_str()),- )?;- Ok(())+ profile_started_at,+ backend_latency_ms,+ response,+ )}fn record_error(&self, decision: &LlmRoutingDecision) -> Result<()> {@@ -489,11 +488,10 @@ impl Profile for LlmRoutingProfile {let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await;match first_result {Ok(response) => {- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;- self.record_success(+ let response = self.record_success(&processed.decision,- &response,- total_latency_ms,+ response,+ profile_started_at,first_backend_latency_ms,)?;let response = self.rprocess(&processed, response).await?;@@ -507,11 +505,10 @@ impl Profile for LlmRoutingProfile {let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await;match retry_result {Ok(response) => {- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;- self.record_success(+ let response = self.record_success(&retry.decision,- &response,- total_latency_ms,+ response,+ profile_started_at,retry_backend_latency_ms,)?;let response = self.rprocess(&retry, response).await?;diff --git a/crates/switchyard-components-v2/src/profiles/passthrough.rs b/crates/switchyard-components-v2/src/profiles/passthrough.rsindex 2becb79f7..8d9edb892 100644--- a/crates/switchyard-components-v2/src/profiles/passthrough.rs+++ b/crates/switchyard-components-v2/src/profiles/passthrough.rs@@ -6,12 +6,12 @@use std::time::Instant;use async_trait::async_trait;-use switchyard_components::stats::usage_from_body;use switchyard_components::StatsAccumulator;use switchyard_core::{ChatResponse, LlmTarget, Result};use crate::backend::{native_target_backend, TargetBackend};use crate::profile_stats_accumulator;+use crate::stats_recording::record_usage_or_wrap_stream;use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse};/// Config for the flatter passthrough profile.@@ -85,15 +85,13 @@ impl Profile for PassthroughProfile {let backend_latency_ms = backend_started_at.elapsed().as_secs_f64() * 1000.0;self.stats.record_success(target_model.to_string(), Some(backend_latency_ms), None)?;- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);- let usage = response.body().map(usage_from_body).unwrap_or_default();- self.stats.record_usage_after_success_attribution(- target_model.to_string(),- usage,- Some(total_latency_ms),- Some(routing_overhead_ms),+ let response = record_usage_or_wrap_stream(+ &self.stats,+ target_model.as_str(),None,+ profile_started_at,+ backend_latency_ms,+ response,)?;let response = self.rprocess(&processed, response).await?;Ok(ProfileResponse::from(response))diff --git a/crates/switchyard-components-v2/src/profiles/random_routing.rs b/crates/switchyard-components-v2/src/profiles/random_routing.rsindex f0fc47386..dea63c67f 100644--- a/crates/switchyard-components-v2/src/profiles/random_routing.rs+++ b/crates/switchyard-components-v2/src/profiles/random_routing.rs@@ -9,12 +9,12 @@ use async_trait::async_trait;use switchyard_components::request_processors::{RandomRoutingDecision, RandomRoutingEngine, RandomRoutingProcessorConfig, RandomRoutingTier,};-use switchyard_components::stats::usage_from_body;use switchyard_components::StatsAccumulator;use switchyard_core::{ChatResponse, LlmTarget, Result, SwitchyardError};use crate::backend::{native_target_backend, TargetBackend};use crate::profile_stats_accumulator;+use crate::stats_recording::record_usage_or_wrap_stream;use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,RoutingMetadata,@@ -167,15 +167,13 @@ impl Profile for RandomRoutingProfile {Some(backend_latency_ms),Some(decision.tier.as_str()),)?;- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);- let usage = response.body().map(usage_from_body).unwrap_or_default();- self.stats.record_usage_after_success_attribution(+ let response = record_usage_or_wrap_stream(+ &self.stats,decision.selected_model.as_str(),- usage,- Some(total_latency_ms),- Some(routing_overhead_ms),Some(decision.tier.as_str()),+ profile_started_at,+ backend_latency_ms,+ response,)?;let response = self.rprocess(&processed, response).await?;Ok(ProfileResponse::with_routing_metadata(@@ -195,8 +193,11 @@ mod tests {use std::sync::{Arc, Mutex};use async_trait::async_trait;+ use futures_util::StreamExt;use serde_json::{json, Value};- use switchyard_core::{BackendFormat, ChatRequest, LlmTargetId, ModelId, SwitchyardError};+ use switchyard_core::{+ BackendFormat, ChatRequest, LlmTargetId, ModelId, StreamEvent, SwitchyardError,+ };use crate::backend::{ProfileBackend, TargetBackend};use crate::RequestMetadata;@@ -214,6 +215,8 @@ mod tests {calls: Arc<Mutex<Vec<ObservedCall>>>,}+ struct StreamingUsageBackend;+#[async_trait]impl ProfileBackend for TestBackend {async fn call(&self, request: &ChatRequest) -> Result<ChatResponse> {@@ -235,6 +238,27 @@ mod tests {}}+ #[async_trait]+ impl ProfileBackend for StreamingUsageBackend {+ async fn call(&self, _request: &ChatRequest) -> Result<ChatResponse> {+ Ok(ChatResponse::OpenAiStream(Box::pin(+ futures_util::stream::iter([+ Ok(StreamEvent::Json(json!({+ "choices": [{"delta": {"content": "ok"}}],+ }))),+ Ok(StreamEvent::Json(json!({+ "choices": [],+ "usage": {+ "prompt_tokens": 3,+ "completion_tokens": 4,+ "total_tokens": 7,+ },+ }))),+ ]),+ )))+ }+ }+fn target(id: &str, model: &str) -> Result<LlmTarget> {let mut target = LlmTarget::new(LlmTargetId::new(id)?, ModelId::new(model)?);target.format = BackendFormat::OpenAi;@@ -385,6 +409,60 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn run_records_streaming_usage_with_selected_random_tier() -> Result<()> {+ let strong = target("strong", "frontier/model")?;+ let weak = target("weak", "cheap/model")?;+ let router_config = RandomRoutingProcessorConfig::new(strong.clone(), weak.clone())+ .with_strong_probability(0.0)?+ .with_rng_seed(Some(7));+ let profile = RandomRoutingProfile {+ router: RandomRoutingEngine::new(router_config)?,+ strong_backend: TargetBackend::new(+ strong,+ Arc::new(TestBackend {+ name: "strong-backend",+ calls: Arc::new(Mutex::new(Vec::new())),+ }),+ ),+ weak_backend: TargetBackend::new(weak, Arc::new(StreamingUsageBackend)),+ stats: StatsAccumulator::new(),+ };++ let response = profile+ .run(profile_input(ChatRequest::openai_chat(json!({+ "model": "client/model",+ "messages": [],+ "stream": true,+ }))))+ .await?;+ let ChatResponse::OpenAiStream(mut stream) = response.response else {+ return Err(SwitchyardError::Other("expected streaming response".into()));+ };+ while let Some(event) = stream.next().await {+ event?;+ }++ let snapshot = profile.stats.snapshot()?;+ assert_eq!(snapshot.total_requests, 1);+ assert_eq!(snapshot.total_tokens.prompt, 3);+ assert_eq!(snapshot.total_tokens.completion, 4);+ assert_eq!(snapshot.total_tokens.total, 7);+ let model = snapshot+ .models+ .get("cheap/model")+ .ok_or_else(|| SwitchyardError::Other("weak model stats should be present".into()))?;+ assert_eq!(model.calls, 1);+ assert_eq!(model.tier.as_deref(), Some("weak"));+ assert_eq!(model.total_tokens, 7);+ let tier = snapshot+ .tiers+ .get("weak")+ .ok_or_else(|| SwitchyardError::Other("weak tier stats should be present".into()))?;+ assert_eq!(tier.total_tokens, 7);+ Ok(())+ }+#[tokio::test]async fn run_disambiguates_duplicate_target_models_without_context_state() -> Result<()> {let (profile, calls) = profile(diff --git a/crates/switchyard-components-v2/src/profiles/stage_router.rs b/crates/switchyard-components-v2/src/profiles/stage_router.rsindex 9a864308e..6aebe709e 100644--- a/crates/switchyard-components-v2/src/profiles/stage_router.rs+++ b/crates/switchyard-components-v2/src/profiles/stage_router.rs@@ -17,6 +17,7 @@ use switchyard_core::{ChatRequest, ChatResponse, LlmTarget, LlmTargetId, Result,use crate::backend::{native_target_backend, TargetBackend};use crate::profile_stats_accumulator;+use crate::stats_recording::record_usage_or_wrap_stream;use crate::{profile_config, Profile, ProfileConfig, ProfileHooks, ProfileInput, ProfileResponse,RoutingMetadata,@@ -483,27 +484,26 @@ impl StageRouterProfile {fn record_success(&self,decision: &StageRouterDecision,- response: &ChatResponse,- total_latency_ms: f64,+ response: ChatResponse,+ profile_started_at: Instant,backend_latency_ms: f64,- ) -> Result<()> {- if let Some(stats) = self.stats_handle() {- stats.record_success(- decision.selected_model.as_str(),- Some(backend_latency_ms),- Some(decision.tier.as_str()),- )?;- let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);- let usage = response.body().map(usage_from_body).unwrap_or_default();- stats.record_usage_after_success_attribution(- decision.selected_model.as_str(),- usage,- Some(total_latency_ms),- Some(routing_overhead_ms),- Some(decision.tier.as_str()),- )?;- }- Ok(())+ ) -> Result<ChatResponse> {+ let Some(stats) = self.stats_handle() else {+ return Ok(response);+ };+ stats.record_success(+ decision.selected_model.as_str(),+ Some(backend_latency_ms),+ Some(decision.tier.as_str()),+ )?;+ record_usage_or_wrap_stream(+ stats,+ decision.selected_model.as_str(),+ Some(decision.tier.as_str()),+ profile_started_at,+ backend_latency_ms,+ response,+ )}fn record_error(&self, decision: &StageRouterDecision) -> Result<()> {@@ -561,11 +561,10 @@ impl Profile for StageRouterProfile {let (first_result, first_backend_latency_ms) = self.call_selected(&processed).await;match first_result {Ok(response) => {- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;- self.record_success(+ let response = self.record_success(&processed.decision,- &response,- total_latency_ms,+ response,+ profile_started_at,first_backend_latency_ms,)?;let response = self.rprocess(&processed, response).await?;@@ -579,11 +578,10 @@ impl Profile for StageRouterProfile {let (retry_result, retry_backend_latency_ms) = self.call_selected(&retry).await;match retry_result {Ok(response) => {- let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;- self.record_success(+ let response = self.record_success(&retry.decision,- &response,- total_latency_ms,+ response,+ profile_started_at,retry_backend_latency_ms,)?;let response = self.rprocess(&retry, response).await?;diff --git a/crates/switchyard-components-v2/src/stats_recording.rs b/crates/switchyard-components-v2/src/stats_recording.rsnew file mode 100644index 000000000..501d30b49--- /dev/null+++ b/crates/switchyard-components-v2/src/stats_recording.rs@@ -0,0 +1,209 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Records provider token usage for buffered and streaming responses.+//!+//! Buffered responses carry usage in the body and can be recorded synchronously.+//! Streaming responses only emit usage inside a later event, so the response+//! stream is wrapped and usage is recorded when the first usage-bearing event+//! flows through — no buffering, no added latency, and the client still sees+//! every event.++use std::time::Instant;++use async_stream::try_stream;+use futures_util::StreamExt;+use switchyard_components::stats::{+ openai_chat_usage_from_stream_event, openai_responses_usage_from_stream_event, usage_from_body,+ AnthropicStreamUsage, StatsAccumulator, TokenUsage,+};+use switchyard_core::{BoxResponseStream, ChatResponse, Result};++/// Records usage from a buffered response, or wraps a streaming response so+/// usage is recorded when its usage-bearing event arrives.+pub(crate) fn record_usage_or_wrap_stream(+ stats: &StatsAccumulator,+ model: &str,+ tier: Option<&str>,+ profile_started_at: Instant,+ backend_latency_ms: f64,+ response: ChatResponse,+) -> Result<ChatResponse> {+ // We report time-to-first-token as latency. Otherwise in streaming case we'd be including+ // the client's processing time.+ let total_latency_ms = profile_started_at.elapsed().as_secs_f64() * 1000.0;+ match response {+ ChatResponse::OpenAiCompletion(_)+ | ChatResponse::OpenAiResponsesCompletion(_)+ | ChatResponse::AnthropicCompletion(_) => {+ let usage = response.body().map(usage_from_body).unwrap_or_default();+ record_usage(+ stats,+ model,+ tier,+ total_latency_ms,+ backend_latency_ms,+ usage,+ )?;+ Ok(response)+ }+ ChatResponse::OpenAiStream(stream) => Ok(ChatResponse::OpenAiStream(wrap_openai_chat(+ stream,+ stats.clone(),+ model.to_string(),+ tier.map(str::to_string),+ total_latency_ms,+ backend_latency_ms,+ ))),+ ChatResponse::OpenAiResponsesStream(stream) => {+ Ok(ChatResponse::OpenAiResponsesStream(wrap_openai_responses(+ stream,+ stats.clone(),+ model.to_string(),+ tier.map(str::to_string),+ total_latency_ms,+ backend_latency_ms,+ )))+ }+ ChatResponse::AnthropicStream(stream) => Ok(ChatResponse::AnthropicStream(wrap_anthropic(+ stream,+ stats.clone(),+ model.to_string(),+ tier.map(str::to_string),+ total_latency_ms,+ backend_latency_ms,+ ))),+ }+}++/// Wraps an OpenAI chat completions stream to record usage stats.+///+/// Usage is typically only present in the last JSON SSE chunk, but we only+/// record the first instance to avoid double counting in case of middle-box weirdness.+///+/// Pass-through, no buffering or dropping.+fn wrap_openai_chat(+ mut stream: BoxResponseStream,+ stats: StatsAccumulator,+ model: String,+ tier: Option<String>,+ total_latency_ms: f64,+ backend_latency_ms: f64,+) -> BoxResponseStream {+ Box::pin(try_stream! {+ let mut committed = false;+ while let Some(event) = stream.next().await {+ let event = event?;+ if !committed {+ if let Some(usage) = openai_chat_usage_from_stream_event(&event) {+ log_stream_record_result(+ record_usage(+ &stats,+ &model,+ tier.as_deref(),+ total_latency_ms,+ backend_latency_ms,+ usage,+ ),+ &model,+ );+ committed = true;+ }+ }+ yield event;+ }+ })+}++/// Wraps an OpenAI responses stream to record usage stats.+fn wrap_openai_responses(+ mut stream: BoxResponseStream,+ stats: StatsAccumulator,+ model: String,+ tier: Option<String>,+ total_latency_ms: f64,+ backend_latency_ms: f64,+) -> BoxResponseStream {+ Box::pin(try_stream! {+ let mut committed = false;+ while let Some(event) = stream.next().await {+ let event = event?;+ if !committed {+ if let Some(usage) = openai_responses_usage_from_stream_event(&event) {+ log_stream_record_result(+ record_usage(+ &stats,+ &model,+ tier.as_deref(),+ total_latency_ms,+ backend_latency_ms,+ usage,+ ),+ &model,+ );+ committed = true;+ }+ }+ yield event;+ }+ })+}++/// Wraps an Anthropic messages stream to record usage stats.+fn wrap_anthropic(+ mut stream: BoxResponseStream,+ stats: StatsAccumulator,+ model: String,+ tier: Option<String>,+ total_latency_ms: f64,+ backend_latency_ms: f64,+) -> BoxResponseStream {+ Box::pin(try_stream! {+ let mut stream_usage = AnthropicStreamUsage::default();+ while let Some(event) = stream.next().await {+ let event = event?;+ if let Some(usage) = stream_usage.observe(&event) {+ log_stream_record_result(+ record_usage(+ &stats,+ &model,+ tier.as_deref(),+ total_latency_ms,+ backend_latency_ms,+ usage,+ ),+ &model,+ );+ }+ yield event;+ }+ })+}++fn record_usage(+ stats: &StatsAccumulator,+ model: &str,+ tier: Option<&str>,+ total_latency_ms: f64,+ backend_latency_ms: f64,+ usage: TokenUsage,+) -> Result<()> {+ let routing_overhead_ms = (total_latency_ms - backend_latency_ms).max(0.0);+ stats.record_usage_after_success_attribution(+ model.to_string(),+ usage,+ Some(total_latency_ms),+ Some(routing_overhead_ms),+ tier,+ )+}++fn log_stream_record_result(result: Result<()>, model: &str) {+ if let Err(error) = result {+ tracing::warn!(+ error = %error,+ model = %model,+ "failed to record stream usage",+ );+ }+}