Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): retain model call spans through completion (#915)
nachiketb-nvidiab9e7cccefc28
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 5c3207a49..30366dbb2 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -2150,7 +2150,7 @@ async fn in_flight_gauge_reads_a_run_parked_on_an_unanswered_routing_call()#[tokio::test]async fn in_flight_gauge_clears_when_a_run_is_abandoned() -> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;- let (_, exporter, provider, _, _) = telemetry();+ let (store, exporter, provider, _, _) = telemetry();const ALGO: &str = "obs-abandoned-algo";const MODEL: &str = "obs-abandoned-model";let algorithm = Arc::new(RoutingCallAlgo {@@ -2195,5 +2195,10 @@ async fn in_flight_gauge_clears_when_a_run_is_abandoned() -> switchyard_libsy::RSome(0),"an abandoned run must not strand the gauge above zero");+ let call_span = find_span(&store.spans(), "libsy.llm_call", "selected_model", MODEL);+ assert_eq!(+ call_span.fields.get("outcome").map(String::as_str),+ Some("error")+ );Ok(())}diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 249805a01..133f8fbe4 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -116,6 +116,8 @@ pub struct CallModel {/// How to send the response back to the algorithm. `None` once the call is recorded.reply: Option<oneshot::Sender<Result<Response>>>,started: Instant,+ // Retain the originating span even if the waiting algorithm is cancelled first.+ span: tracing::Span,}impl CallModel {@@ -124,6 +126,9 @@ impl CallModel {/// can only be fulfilled once.pub fn respond(mut self, result: Result<Response>) -> Result<()> {self.record(result.is_ok());+ if let Ok(response) = &result {+ observability::record_llm_response(response, &self.span);+ }self.reply.take().ok_or(DriverError::ResponseDropped)?@@ -149,6 +154,8 @@ impl CallModel {self.started.elapsed(),is_ok,);+ self.span+ .record("outcome", if is_ok { "ok" } else { "error" });}}@@ -331,9 +338,9 @@ impl Driver {/// Errors if the stream is closed or the call failed./// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as/// the algorithm observes it (host queueing/serving included; a streamed- /// response resolves when its stream handle arrives). The host records call metrics- /// through [`CallModel::respond`] or [`CallModel::fail`]; outcome and token usage- /// are recorded on the span when the promise resolves. The provider call itself is the+ /// response resolves when its stream handle arrives). The host records call metrics,+ /// outcome, and buffered token usage when it completes or drops the call, even if the+ /// waiting algorithm has been cancelled. The provider call itself is the/// host's, and is instrumented by whoever makes it.pub async fn call_model(&self, request: Request, models: Vec<ModelId>) -> Result<Response> {self.call_model_with_error_recovery(request, models, false)@@ -375,19 +382,15 @@ impl Driver {recover_errors,reply: Some(reply),started,+ span: tracing::Span::current(),};- let result = async {- self.step_tx- .send(Ok(Step::CallModel(Box::new(call))))- .await- .map_err(|_| DriverError::StreamClosed)?;- response- .await- .map_err(|_| LibsyError::from(DriverError::ResponseDropped))?- }- .await;- observability::record_llm_call_span(&result, &tracing::Span::current());- result+ self.step_tx+ .send(Ok(Step::CallModel(Box::new(call))))+ .await+ .map_err(|_| DriverError::StreamClosed)?;+ response+ .await+ .map_err(|_| LibsyError::from(DriverError::ResponseDropped))?}/// Override the request's model with `model` and wait for the host to reply.diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 4775256c3..6cacfd4ca 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -256,15 +256,10 @@ pub(crate) fn record_llm_call(.record(duration.as_secs_f64() * 1000.0, &call_attributes);}-/// Records the outcome and token fields on the algorithm's call span.-pub(crate) fn record_llm_call_span(result: &Result<Response>, span: &Span) {- span.record("outcome", outcome_value(result));- if let Ok(response) = result {- // Token usage exists only once a response is buffered; a streamed- // response resolves before its usage is known, so none is recorded.- let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {- return;- };+/// Records buffered token usage on the algorithm's call span.+pub(crate) fn record_llm_response(response: &Response, span: &Span) {+ // A streamed response resolves before its usage is known.+ if let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) {record_call_usage(usage, span);}}