Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(metrics): record streamed calls by their final outcome (#772)

nachiketb-nvidia328591ddba8e
diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rs
index 76730ebfe..961c7b20c 100644
--- a/crates/libsy-llm-client/src/metrics.rs
+++ b/crates/libsy-llm-client/src/metrics.rs
@@ -3,7 +3,9 @@
//! Metric labelling inherited from Python
-use std::time::Duration;
+use std::time::{Duration, Instant};
+
+use futures::{StreamExt, stream};
use std::{
sync::OnceLock,
sync::atomic::{AtomicU64, Ordering},
@@ -12,7 +14,7 @@ use std::{
use opentelemetry::metrics::ObservableGauge;
use opentelemetry::{KeyValue, global};
use switchyard_libsy::Result;
-use switchyard_protocol::{ModelId, Response};
+use switchyard_protocol::{LlmResponse, LlmResponseChunk, ModelId, Response};
static TOTAL_REQUESTS: AtomicU64 = AtomicU64::new(0);
static TOTAL_ERRORS: AtomicU64 = AtomicU64::new(0);
@@ -116,17 +118,88 @@ pub(crate) fn record_routing_overhead(algorithm: &str, overhead: Duration) {
);
}
+/// Records the final outcome, retaining ownership until a streamed answer ends or is dropped.
+pub(crate) fn observe_routed_request(
+ algorithm: &str,
+ selected_model: &ModelId,
+ answer_duration: Option<Duration>,
+ result: Result<Response>,
+) -> Result<Response> {
+ let mut call = RoutedCallMetrics {
+ algorithm: algorithm.to_owned(),
+ model: selected_model.clone(),
+ duration: answer_duration,
+ started: Instant::now(),
+ outcome: Some(result.is_ok()),
+ };
+ result.map(|mut response| {
+ if let LlmResponse::Stream(stream) = response.llm_response {
+ call.outcome = None;
+ response.llm_response = LlmResponse::Stream(Box::pin(stream::unfold(
+ (stream, call),
+ |(mut stream, mut call)| async move {
+ let Some(item) = stream.next().await else {
+ call.outcome.get_or_insert(true);
+ return None;
+ };
+ match &item {
+ Err(_) => call.outcome = Some(false),
+ Ok(event) => {
+ for chunk in event.normalized() {
+ match chunk {
+ LlmResponseChunk::StreamError { .. }
+ | LlmResponseChunk::DecodeError { .. } => {
+ call.outcome = Some(false)
+ }
+ LlmResponseChunk::MessageStop { .. } => {
+ call.outcome.get_or_insert(true);
+ }
+ _ => {}
+ }
+ }
+ }
+ }
+ Some((item, (stream, call)))
+ },
+ )));
+ }
+ response
+ })
+}
+
+struct RoutedCallMetrics {
+ algorithm: String,
+ model: ModelId,
+ duration: Option<Duration>,
+ started: Instant,
+ // Dropping an open stream is a failure; a terminal message permits a successful drop.
+ outcome: Option<bool>,
+}
+
+impl Drop for RoutedCallMetrics {
+ fn drop(&mut self) {
+ let is_success = self.outcome.unwrap_or(false);
+ let duration = self
+ .duration
+ .map(|duration| duration + self.started.elapsed());
+ if let Some(duration) = duration {
+ record_answer_call(&self.algorithm, &self.model, duration, is_success);
+ }
+ record_routed_request(&self.model, duration, is_success);
+ }
+}
+
/// Records one terminal model call made after routing, preserving the libsy call metric surface.
pub(crate) fn record_answer_call(
algorithm: &str,
selected_model: &ModelId,
duration: Duration,
- result: &Result<Response>,
+ is_success: bool,
) {
let attributes = [
KeyValue::new("algorithm", algorithm.to_string()),
KeyValue::new("selected_model", selected_model.to_string()),
- KeyValue::new("outcome", if result.is_ok() { "ok" } else { "error" }),
+ KeyValue::new("outcome", if is_success { "ok" } else { "error" }),
];
let meter = global::meter("switchyard");
meter
@@ -143,12 +216,12 @@ pub(crate) fn record_answer_call(
pub(crate) fn record_routed_request(
selected_model: &ModelId,
answer_duration: Option<Duration>,
- result: &Result<Response>,
+ is_success: bool,
) {
TOTAL_REQUESTS.fetch_add(1, Ordering::Relaxed);
let attributes = [KeyValue::new("model", selected_model.to_string())];
let meter = global::meter("switchyard");
- if result.is_ok() {
+ if is_success {
meter
.u64_counter("switchyard.requests")
.build()
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index ef8324578..d4dcc5de7 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -108,17 +108,16 @@ pub async fn run(
)
.await;
let answer_duration = answer_started.elapsed();
- metrics::record_answer_call(
- &algorithm_name,
- &selected_model_id,
- answer_duration,
- &result,
- );
(result, Some(answer_duration))
};
let result =
result.and_then(|response| clients.remember_state_owner(&outcome.request, response));
- metrics::record_routed_request(&selected_model_id, answer_duration, &result);
+ let result = metrics::observe_routed_request(
+ &algorithm_name,
+ &selected_model_id,
+ answer_duration,
+ result,
+ );
if let Some(observer) = &observer {
observer(RunObservation::RoutingOverhead(overhead));
}
diff --git a/crates/switchyard-server/src/usage_metrics.rs b/crates/switchyard-server/src/usage_metrics.rs
index 158b58779..045f51c86 100644
--- a/crates/switchyard-server/src/usage_metrics.rs
+++ b/crates/switchyard-server/src/usage_metrics.rs
@@ -107,10 +107,6 @@ pub(crate) fn observe(
// Records a terminal stream failure after the routed call was already counted.
fn record_stream_error(stats: &StatsAccumulator, model: &str) {
stats.record_response_error(model);
- global::meter("switchyard")
- .u64_counter("switchyard.errors")
- .build()
- .add(1, &attributes(model));
}
pub(crate) fn token_usage(usage: &Usage) -> TokenUsage {
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 76ecbc1a1..f05094f7a 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -4299,6 +4299,8 @@ async fn streaming_error_records_error_without_usage_or_latency() -> TestResult
let after = send(&app, "GET", "/metrics", None).await?;
let after = after.text()?;
for name in [
+ "switchyard_requests_total",
+ "switchyard_model_call_latency_ms_count",
"switchyard_prompt_tokens_total",
"switchyard_completion_tokens_total",
"switchyard_cached_tokens_total",
@@ -4321,6 +4323,7 @@ async fn streaming_error_records_error_without_usage_or_latency() -> TestResult
),
Some(1.0)
);
+ assert!(metric_delta(before, after, "switchyard_total_errors", &[]).unwrap_or_default() >= 1.0);
let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
assert_eq!(stats["total_requests"], 1);
assert_eq!(stats["total_errors"], 1);