Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(metrics): record streamed calls by their final outcome (#772)
nachiketb-nvidia328591ddba8e
diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rsindex 76730ebfe..961c7b20c 100644--- a/crates/libsy-llm-client/src/metrics.rs+++ b/crates/libsy-llm-client/src/metrics.rs@@ -3,7 +3,9 @@//! Metric labelling inherited from Python-use std::time::Duration;+use std::time::{Duration, Instant};++use futures::{StreamExt, stream};use std::{sync::OnceLock,sync::atomic::{AtomicU64, Ordering},@@ -12,7 +14,7 @@ use std::{use opentelemetry::metrics::ObservableGauge;use opentelemetry::{KeyValue, global};use switchyard_libsy::Result;-use switchyard_protocol::{ModelId, Response};+use switchyard_protocol::{LlmResponse, LlmResponseChunk, ModelId, Response};static TOTAL_REQUESTS: AtomicU64 = AtomicU64::new(0);static TOTAL_ERRORS: AtomicU64 = AtomicU64::new(0);@@ -116,17 +118,88 @@ pub(crate) fn record_routing_overhead(algorithm: &str, overhead: Duration) {);}+/// Records the final outcome, retaining ownership until a streamed answer ends or is dropped.+pub(crate) fn observe_routed_request(+ algorithm: &str,+ selected_model: &ModelId,+ answer_duration: Option<Duration>,+ result: Result<Response>,+) -> Result<Response> {+ let mut call = RoutedCallMetrics {+ algorithm: algorithm.to_owned(),+ model: selected_model.clone(),+ duration: answer_duration,+ started: Instant::now(),+ outcome: Some(result.is_ok()),+ };+ result.map(|mut response| {+ if let LlmResponse::Stream(stream) = response.llm_response {+ call.outcome = None;+ response.llm_response = LlmResponse::Stream(Box::pin(stream::unfold(+ (stream, call),+ |(mut stream, mut call)| async move {+ let Some(item) = stream.next().await else {+ call.outcome.get_or_insert(true);+ return None;+ };+ match &item {+ Err(_) => call.outcome = Some(false),+ Ok(event) => {+ for chunk in event.normalized() {+ match chunk {+ LlmResponseChunk::StreamError { .. }+ | LlmResponseChunk::DecodeError { .. } => {+ call.outcome = Some(false)+ }+ LlmResponseChunk::MessageStop { .. } => {+ call.outcome.get_or_insert(true);+ }+ _ => {}+ }+ }+ }+ }+ Some((item, (stream, call)))+ },+ )));+ }+ response+ })+}++struct RoutedCallMetrics {+ algorithm: String,+ model: ModelId,+ duration: Option<Duration>,+ started: Instant,+ // Dropping an open stream is a failure; a terminal message permits a successful drop.+ outcome: Option<bool>,+}++impl Drop for RoutedCallMetrics {+ fn drop(&mut self) {+ let is_success = self.outcome.unwrap_or(false);+ let duration = self+ .duration+ .map(|duration| duration + self.started.elapsed());+ if let Some(duration) = duration {+ record_answer_call(&self.algorithm, &self.model, duration, is_success);+ }+ record_routed_request(&self.model, duration, is_success);+ }+}+/// Records one terminal model call made after routing, preserving the libsy call metric surface.pub(crate) fn record_answer_call(algorithm: &str,selected_model: &ModelId,duration: Duration,- result: &Result<Response>,+ is_success: bool,) {let attributes = [KeyValue::new("algorithm", algorithm.to_string()),KeyValue::new("selected_model", selected_model.to_string()),- KeyValue::new("outcome", if result.is_ok() { "ok" } else { "error" }),+ KeyValue::new("outcome", if is_success { "ok" } else { "error" }),];let meter = global::meter("switchyard");meter@@ -143,12 +216,12 @@ pub(crate) fn record_answer_call(pub(crate) fn record_routed_request(selected_model: &ModelId,answer_duration: Option<Duration>,- result: &Result<Response>,+ is_success: bool,) {TOTAL_REQUESTS.fetch_add(1, Ordering::Relaxed);let attributes = [KeyValue::new("model", selected_model.to_string())];let meter = global::meter("switchyard");- if result.is_ok() {+ if is_success {meter.u64_counter("switchyard.requests").build()diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex ef8324578..d4dcc5de7 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -108,17 +108,16 @@ pub async fn run().await;let answer_duration = answer_started.elapsed();- metrics::record_answer_call(- &algorithm_name,- &selected_model_id,- answer_duration,- &result,- );(result, Some(answer_duration))};let result =result.and_then(|response| clients.remember_state_owner(&outcome.request, response));- metrics::record_routed_request(&selected_model_id, answer_duration, &result);+ let result = metrics::observe_routed_request(+ &algorithm_name,+ &selected_model_id,+ answer_duration,+ result,+ );if let Some(observer) = &observer {observer(RunObservation::RoutingOverhead(overhead));}diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rsindex 158b58779..045f51c86 100644--- a/crates/switchyard-server/src/usage_metrics.rs+++ b/crates/switchyard-server/src/usage_metrics.rs@@ -107,10 +107,6 @@ pub(crate) fn observe(// Records a terminal stream failure after the routed call was already counted.fn record_stream_error(stats: &StatsAccumulator, model: &str) {stats.record_response_error(model);- global::meter("switchyard")- .u64_counter("switchyard.errors")- .build()- .add(1, &attributes(model));}pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 76ecbc1a1..f05094f7a 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -4299,6 +4299,8 @@ async fn streaming_error_records_error_without_usage_or_latency() -> TestResultlet after = send(&app, "GET", "/metrics", None).await?;let after = after.text()?;for name in [+ "switchyard_requests_total",+ "switchyard_model_call_latency_ms_count","switchyard_prompt_tokens_total","switchyard_completion_tokens_total","switchyard_cached_tokens_total",@@ -4321,6 +4323,7 @@ async fn streaming_error_records_error_without_usage_or_latency() -> TestResult),Some(1.0));+ assert!(metric_delta(before, after, "switchyard_total_errors", &[]).unwrap_or_default() >= 1.0);let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;assert_eq!(stats["total_requests"], 1);assert_eq!(stats["total_errors"], 1);