Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat: Move observation from libsy to libsy-llm-client (#332)
Graham Kinge8390d43e88c
diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.tomlindex 0bba91472..9a9ca6e0a 100644--- a/crates/libsy-llm-client/Cargo.toml+++ b/crates/libsy-llm-client/Cargo.toml@@ -25,6 +25,7 @@ async-trait.workspace = truefutures.workspace = truefutures-util.workspace = trueopentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }+parking_lot.workspace = truehttp.workspace = truehttpdate.workspace = trueserde_json.workspace = true@@ -35,7 +36,6 @@ tracing-opentelemetry.workspace = true[dev-dependencies]# SDK + in-memory exporter to assert what the observability layer records.opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }-parking_lot.workspace = truethiserror.workspace = truetracing-subscriber.workspace = truewiremock = "0.6"diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex f64259c3d..54c9737b6 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -23,7 +23,7 @@ use tracing::Instrument;use crate::backend::Backend;use crate::error::{LlmClientError, Result};-use crate::metrics::{is_retryable_http_status, record_upstream_attempt};+use crate::metrics;use crate::raw::RawResponse;// TODO: Why is this here? What does it do?@@ -287,7 +287,7 @@ impl TranslatingLlmClient {let response = match builder.send().await {Ok(response) => response,Err(error) => {- record_upstream_attempt(None);+ metrics::record_upstream_attempt(None);return Err(AttemptFailure {error: convert_reqwest_error(error),status: None,@@ -299,13 +299,13 @@ impl TranslatingLlmClient {if status.is_success() {if streaming {// Streaming body failures happen after the retry boundary.- record_upstream_attempt(Some(status.as_u16()));+ metrics::record_upstream_attempt(Some(status.as_u16()));return Ok(EncodedResponse::Streaming(response));}let body = match response.bytes().await {Ok(body) => body,Err(error) => {- record_upstream_attempt(None);+ metrics::record_upstream_attempt(None);return Err(AttemptFailure {error: convert_reqwest_error(error),status: Some(status.as_u16()),@@ -313,7 +313,7 @@ impl TranslatingLlmClient {});}};- record_upstream_attempt(Some(status.as_u16()));+ metrics::record_upstream_attempt(Some(status.as_u16()));return Ok(EncodedResponse::Buffered {status: status.as_u16(),body: body.to_vec(),@@ -324,7 +324,7 @@ impl TranslatingLlmClient {let body = match response.text().await {Ok(body) => body,Err(error) => {- record_upstream_attempt(None);+ metrics::record_upstream_attempt(None);return Err(AttemptFailure {error: convert_reqwest_error(error),status: Some(status.as_u16()),@@ -332,7 +332,7 @@ impl TranslatingLlmClient {});}};- record_upstream_attempt(Some(status.as_u16()));+ metrics::record_upstream_attempt(Some(status.as_u16()));let error =if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {LlmClientError::ContextWindowExceeded {@@ -563,7 +563,9 @@ impl AttemptFailure {fn is_retryable(&self) -> bool {match &self.error {LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,- LlmClientError::UpstreamHttp { status, .. } => is_retryable_http_status(*status),+ LlmClientError::UpstreamHttp { status, .. } => {+ metrics::is_retryable_http_status(*status)+ }_ => false,}}diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rsindex f1c8a462d..e10a17dd6 100644--- a/crates/libsy-llm-client/src/lib.rs+++ b/crates/libsy-llm-client/src/lib.rs@@ -21,12 +21,14 @@ pub mod client;pub mod error;pub mod metrics;mod observability;+mod observation;pub mod raw;pub mod run;pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};pub use client::{ModelConfig, TranslatingLlmClient};pub use error::{LlmClientError, Result};+pub use observation::{LlmCallObservation, RunObservation, RunObserver};pub use raw::RawResponse;pub use run::{ClientRouter, run};pub use switchyard_translation::RawEventStream;diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rsindex 672d8dd5f..582bb5f6b 100644--- a/crates/libsy-llm-client/src/metrics.rs+++ b/crates/libsy-llm-client/src/metrics.rs@@ -3,6 +3,8 @@//! Metric labelling inherited from Python+use std::time::Duration;+use opentelemetry::{KeyValue, global};pub(crate) const fn is_retryable_http_status(status: u16) -> bool {@@ -59,6 +61,26 @@ pub(crate) fn record_upstream_attempt(status: Option<u16>) {);}+/// Records what routing cost on top of the call that served the run: classifier+/// calls, target resolution, and decision publishing.+pub(crate) fn record_routing_overhead(+ algorithm: &str,+ run: Duration,+ call_duration: Duration,+) -> Duration {+ // Saturating: the two clocks start a moment apart, so a run that is all+ // routed call can come out fractionally negative.+ let overhead = run.saturating_sub(call_duration);+ global::meter("switchyard")+ .f64_histogram("switchyard.routing_overhead_ms")+ .build()+ .record(+ overhead.as_secs_f64() * 1000.0,+ &[KeyValue::new("algorithm", algorithm.to_string())],+ );+ overhead+}+#[cfg(test)]mod tests {use super::*;diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rsindex 1767ffad8..58270cc03 100644--- a/crates/libsy-llm-client/src/observability.rs+++ b/crates/libsy-llm-client/src/observability.rs@@ -1,18 +1,6 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! `tracing` span fields for the provider call [`run`](crate::run) makes.-//!-//! libsy instruments the orchestration layer (`libsy.run`, `libsy.llm_call`) and owns the-//! `switchyard.*` metrics. This module owns the other half: the `libsy.client_call` span that-//! covers the one HTTP call a routed request performs, recorded as OpenTelemetry gen_ai-//! semantic-convention attributes.-//!-//! Nothing here records a metric — these are span fields only, written through the `tracing`-//! facade so the host's subscriber decides where they go. A streamed response resolves before-//! its usage is known, so [`observe_client_call`] wraps the stream and keeps the span alive-//! until it drains, errors, or is abandoned.-use std::borrow::Cow;use std::pin::Pin;use std::task::{Context as TaskContext, Poll};diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rsnew file mode 100644index 000000000..c6b955081--- /dev/null+++ b/crates/libsy-llm-client/src/observation.rs@@ -0,0 +1,42 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++//! Request-scoped observations emitted while serving an algorithm run.++use std::sync::Arc;+use std::time::Duration;++use switchyard_protocol::Usage;++/// One completed model call observed at the algorithm offload boundary.+#[derive(Clone, Debug)]+pub struct LlmCallObservation {+ /// Model selected for the completed call.+ pub selected_model: String,+ /// Routing tier attached to the selected model, when present.+ pub tier: Option<String>,+ /// Whether this was the routed backend call rather than classifier or judge overhead.+ pub is_routed: bool,+ /// Whether the call completed successfully.+ pub is_success: bool,+ /// Time spent waiting for the model call to resolve.+ pub duration: Duration,+ /// Normalized usage for a buffered successful response.+ pub usage: Option<Usage>,+}++/// One request-scoped observation emitted by the algorithm runner.+#[derive(Clone, Debug)]+pub enum RunObservation {+ /// A completed model call.+ LlmCall(LlmCallObservation),+ /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.+ RoutingOverhead(Duration),+}++/// Request-scoped callback for algorithm-run observations.+///+/// The runner invokes this callback inline while resolving observations. Several+/// runs may invoke the same observer concurrently, so implementations must be+/// thread-safe, fast, and non-blocking.+pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex efa500ab0..c3655a07b 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -13,13 +13,14 @@use std::collections::HashMap;use std::sync::Arc;+use std::time::{Duration, Instant};-use switchyard_libsy::{- Algorithm, CallLlmRequest, LibsyError, Result, RunObserver, algorithm_label, drive,-};+use parking_lot::Mutex;+use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Result, algorithm_label, drive};use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response, RoutedLlmClient};-use crate::observability;+use crate::observation::{LlmCallObservation, RunObservation, RunObserver};+use crate::{metrics, observability};/// Run one request to completion, serving every offloaded model call with `client`.///@@ -41,10 +42,61 @@ pub async fn run(request: Request,observer: Option<RunObserver>,) -> Result<(Vec<Arc<dyn Decision>>, Response)> {- drive(algorithm, ctx, request, observer, move |call| {- serve(clients.clone(), call)+ let algorithm_name = algorithm.name().to_string();+ // The output from `serve` goes in here: when each successful routed call was in+ // flight. Everything else the run spent time on is routing overhead.+ let routed_calls = Arc::new(Mutex::new(RoutedCallWindows::default()));+ let run_started = Instant::now();+ let result = drive(algorithm, ctx, request, {+ let observer = observer.clone();+ let routed_calls = Arc::clone(&routed_calls);+ move |call| {+ serve(+ clients.clone(),+ call,+ observer.clone(),+ Arc::clone(&routed_calls),+ )+ }})- .await+ .await?;+ if let Some(served) = routed_calls.lock().served() {+ let overhead =+ metrics::record_routing_overhead(&algorithm_name, run_started.elapsed(), served);+ if let Some(observer) = observer {+ observer(RunObservation::RoutingOverhead(overhead));+ }+ }+ Ok(result)+}++/// The wall-clock windows during which a successful routed call was in flight.+/// An algorithm can make multiple overlapping calls. This is the union of all of them.+#[derive(Default)]+struct RoutedCallWindows(Vec<(Instant, Instant)>);++impl RoutedCallWindows {+ /// Record one completed routed call.+ fn record(&mut self, started: Instant, ended: Instant) {+ self.0.push((started, ended));+ }++ /// Total time at least one routed call was in flight, merging overlapping windows.+ fn served(&mut self) -> Option<Duration> {+ self.0.sort_unstable_by_key(|(started, _)| *started);+ // Sweep the windows in start order, advancing a cursor along the timeline and+ // counting only the time each window covers that the cursor has not reached.+ let mut covered = self.0.first()?.0;+ let mut total = Duration::ZERO;+ for &(started, ended) in &self.0 {+ covered = covered.max(started);+ if ended > covered {+ total += ended - covered;+ covered = ended;+ }+ }+ Some(total)+ }}/// Serve one offloaded call. A failed *model* call is forwarded to the algorithm via@@ -88,7 +140,13 @@ pub async fn run(error = tracing::field::Empty,))]-async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {+async fn serve(+ clients: ClientRouter,+ call: CallLlmRequest,+ observer: Option<RunObserver>,+ // Output parameter because `drive` takes a function that returns a plain `Result<()>`.+ routed_calls: Arc<Mutex<RoutedCallWindows>>,+) -> Result<()> {let span = tracing::Span::current();observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);if let Some(tier) = call.get_decision().routing_tier() {@@ -105,7 +163,13 @@ async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {}let routed = call.get_routed().clone();let target = routed.decision.selected_model().to_string();- let result = match clients.route(&target) {+ let tier = call.get_decision().routing_tier().map(str::to_string);+ let is_routed = call.get_decision().is_routed_call();+ // Resolved before the clock starts: picking the client is Switchyard's work, not+ // the provider's, so it belongs in the routing overhead.+ let client = clients.route(&target);+ let started = Instant::now();+ let result = match client {Ok(client) => {client.call(routed.ctx, routed.request, routed.decision)@@ -114,7 +178,28 @@ async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {Err(error) => Err(error),}.map_err(|source| LibsyError::client_call(target, source));+ let ended = Instant::now();+ let duration = ended - started;+let result = observability::observe_client_call(result);+ if let Some(observer) = observer {+ observer(RunObservation::LlmCall(LlmCallObservation {+ selected_model: call.get_decision().selected_model().to_string(),+ tier,+ is_routed,+ is_success: result.is_ok(),+ duration,+ usage: result+ .as_ref()+ .ok()+ .and_then(|response| response.llm_response.as_agg())+ .map(|response| response.usage.clone()),+ }));+ }+ if is_routed && result.is_ok() {+ routed_calls.lock().record(started, ended);+ }+call.respond(result)}diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 864c95f56..721b722d0 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -36,7 +36,7 @@ use switchyard_libsy::{Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,RoutedRequest, Step, TaskClassifierConfig,};-use switchyard_llm_client::ClientRouter;+use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};use switchyard_protocol::{Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,};@@ -843,6 +843,50 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lOk(())}+#[tokio::test]+async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::Result<()> {+ let _guard = serialize_test().lock().await;+ let observations = Arc::new(Mutex::new(Vec::new()));+ let observed = Arc::clone(&observations);+ let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));+ const ALGO: &str = "observed-run-algo";+ const MODEL: &str = "observed-run-model";+ let client = Arc::new(UsageClient {+ usage: Usage::default(),+ }) as Arc<dyn RoutedLlmClient>;++ let (_, response) = switchyard_llm_client::run(+ algo(ALGO, MODEL),+ ClientRouter::single(client),+ Context::default(),+ request_with_metadata("observed-session", "observed-correlation"),+ Some(observer),+ )+ .await?;++ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(|response| response.model.as_deref()),+ Some(Some(MODEL))+ );+ let observations = observations.lock();+ assert_eq!(observations.len(), 2);+ let RunObservation::LlmCall(observation) = &observations[0] else {+ return Err(test_error("expected an LLM call observation"));+ };+ assert_eq!(observation.selected_model, MODEL);+ assert!(observation.is_routed);+ assert!(observation.is_success);+ assert!(observation.usage.is_some());+ assert!(matches!(+ observations[1],+ RunObservation::RoutingOverhead(_)+ ));+ Ok(())+}+/// A streamed response keeps the client span available until terminal usage arrives.struct StreamingUsageClient;@@ -1006,7 +1050,6 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::let stream = algo(ALGO, MODEL).run_stream(Context::default(),request_with_metadata("obs-session-2", "obs-corr-2"),- None,);tokio::pin!(stream);diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 331943354..60a7c8808 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -463,7 +463,7 @@ mod tests {use crate::core::classifier::Classification;use crate::{AffinityRouter, SystemPromptProcessor, TargetPrompts};- use crate::core::testing::{Serve, drive, echo, reply};+ use crate::core::testing::{Serve, echo, reply, test_drive};use switchyard_protocol::{LlmClientError, LlmRequest, Message, Metadata, Role, completion_text, text_request,};@@ -562,7 +562,7 @@ mod tests {recorder: &Arc<PromptRecorder>,router: FallThrough,) -> Result<RecordedCall> {- drive(+ test_drive(Arc::new(router),Context::default(),Request {@@ -648,7 +648,8 @@ mod tests {whereS: Default + Send + 'static,{- let (trace, response) = drive(router.clone(), Context::default(), request, serve).await?;+ let (trace, response) =+ test_drive(router.clone(), Context::default(), request, serve).await?;let text = response.llm_response.into_agg()@@ -958,7 +959,7 @@ mod tests {);let mut ctx = Context::default();ctx.exclude_target("weak");- let (trace, response) = drive(router, ctx, request(), echo()).await?;+ let (trace, response) = test_drive(router, ctx, request(), echo()).await?;let text = response.llm_response.into_agg()@@ -1252,7 +1253,7 @@ mod tests {..request()};- let result = drive(router.clone(), Context::default(), final_request, echo()).await;+ let result = test_drive(router.clone(), Context::default(), final_request, echo()).await;assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));let states = routerdiff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex d75748d14..d5be7492f 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -959,7 +959,7 @@ mod tests {};use crate::algorithms::util::llm_judge::Judge;- use crate::core::testing::{Serve, drive, reply};+ use crate::core::testing::{Serve, reply, test_drive};use switchyard_protocol::{Context, LlmResponse, Response};const TEST_THRESHOLD: f64 = 0.5;@@ -1128,7 +1128,7 @@ mod tests {async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {let router = router()?;- let (trace, response) = drive(+ let (trace, response) = test_drive(router,Context::default(),classify_request(),@@ -1150,14 +1150,14 @@ mod tests {let router = router()?;let request = classify_request;- drive(+ test_drive(router.clone(),Context::default(),request(),recorder.serve(),).await?;- drive(+ test_drive(router.clone(),Context::default(),request(),@@ -1188,7 +1188,7 @@ mod tests {},})?);- drive(+ test_drive(router,Context::default(),classify_request(),@@ -1217,7 +1217,7 @@ mod tests {},})?);- drive(+ test_drive(router,Context::default(),classify_request(),@@ -1248,14 +1248,14 @@ mod tests {})?);let session_request = classify_session_request;- drive(+ test_drive(router.clone(),Context::default(),session_request(),recorder.serve(),).await?;- drive(+ test_drive(router.clone(),Context::default(),session_request(),@@ -1285,14 +1285,14 @@ mod tests {},})?);- drive(+ test_drive(router.clone(),Context::default(),classify_request(),recorder.serve(),).await?;- drive(+ test_drive(router.clone(),Context::default(),classify_follow_up_request(),@@ -1860,7 +1860,7 @@ mod tests {let model = Queue::new(["efficient answer"]);let router = escalation_router()?;- let (trace, response) = drive(+ let (trace, response) = test_drive(router,Context::default(),classify_request(),@@ -1895,7 +1895,7 @@ mod tests {max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,})?);- drive(+ test_drive(router,Context::default(),classify_request(),@@ -1917,7 +1917,7 @@ mod tests {let model = Queue::new(["efficient draft", "capable answer"]);let router = escalation_router()?;- let (trace, response) = drive(+ let (trace, response) = test_drive(router,Context::default(),classify_request(),@@ -1942,14 +1942,14 @@ mod tests {let router = escalation_router()?;let session_request = classify_session_request();- drive(+ test_drive(router.clone(),Context::default(),session_request.clone(),queued(Arc::clone(&model), Arc::clone(&judge)),).await?;- let (trace, _) = drive(+ let (trace, _) = test_drive(router.clone(),Context::default(),session_request,@@ -1981,7 +1981,7 @@ mod tests {};let (trace, response) =- drive(router, Context::default(), classify_request(), serve).await?;+ test_drive(router, Context::default(), classify_request(), serve).await?;assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));assert_eq!(diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rsindex 66e2b9cdb..853c297a1 100644--- a/crates/libsy/src/algorithms/noop.rs+++ b/crates/libsy/src/algorithms/noop.rs@@ -79,7 +79,7 @@ mod tests {use switchyard_protocol::{LlmRequest, Message, Role};use super::*;- use crate::core::testing::{drive, echo};+ use crate::core::testing::{echo, test_drive};#[tokio::test]async fn test_noop_algo() -> Result<()> {@@ -97,7 +97,7 @@ mod tests {// `Noop` synthesizes its own response and never offloads a call, so `echo` is// never reached.let a: Arc<dyn Algorithm> = Arc::new(Noop {});- let (decisions, response) = drive(a, Context::default(), request, echo()).await?;+ let (decisions, response) = test_drive(a, Context::default(), request, echo()).await?;let Some(decision) = decisions.first() else {panic!("Expected exactly one Decision");};diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex 85817ba4d..c8c74faeb 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -73,7 +73,7 @@ mod tests {use super::Passthrough;use crate::core::algorithm::{Algorithm, LlmTarget};- use crate::core::testing::{drive, echo};+ use crate::core::testing::{echo, test_drive};use switchyard_protocol::{Context, Request, completion_text, text_request};#[tokio::test]@@ -87,7 +87,7 @@ mod tests {let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {semantic_name: MODEL_ID.to_string(),}));- let (trace, response) = drive(algorithm, Context::default(), request, echo()).await?;+ let (trace, response) = test_drive(algorithm, Context::default(), request, echo()).await?;assert_eq!(responsediff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex caa3fecc5..a8017cad7 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -182,7 +182,7 @@ mod tests {use crate::DriverError;use crate::algorithms::util::affinity::AffinityRouter;use crate::core::algorithm::LlmTarget;- use crate::core::testing::{drive, echo};+ use crate::core::testing::{echo, test_drive};use switchyard_protocol::{Request, Signals};fn request() -> Request {@@ -225,7 +225,7 @@ mod tests {let mut selected = Vec::with_capacity(count);for _ in 0..count {let (_, response) =- drive(algorithm.clone(), Context::default(), request(), echo()).await?;+ test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;selected.push(response.llm_response@@ -240,7 +240,8 @@ mod tests {#[tokio::test]async fn single_target_is_always_selected_and_called() -> Result<()> {let algorithm = shared_algorithm(&["only/model"])?;- let (trace, response) = drive(algorithm, Context::default(), request(), echo()).await?;+ let (trace, response) =+ test_drive(algorithm, Context::default(), request(), echo()).await?;assert_eq!(response@@ -262,7 +263,7 @@ mod tests {for _ in 0..50 {let (trace, response) =- drive(algorithm.clone(), Context::default(), request(), echo()).await?;+ test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;let selected = response.llm_response.as_agg()@@ -284,7 +285,7 @@ mod tests {for _ in 0..100 {let (_, response) =- drive(algorithm.clone(), Context::default(), request(), echo()).await?;+ test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;seen.insert(response.llm_response@@ -348,7 +349,7 @@ mod tests {.with_classifier(random),);- let (_, first) = drive(+ let (_, first) = test_drive(algorithm.clone(),Context::default(),request_for_session("session-1"),@@ -373,7 +374,7 @@ mod tests {Some(selected.to_string()));- let (_, second) = drive(+ let (_, second) = test_drive(algorithm,Context::default(),request_for_session("session-1"),@@ -431,7 +432,7 @@ mod tests {#[tokio::test]async fn decision_is_inspectable_and_downcasts() -> Result<()> {let algorithm = shared_algorithm(&["only/model"])?;- let (trace, _) = drive(algorithm, Context::default(), request(), echo()).await?;+ let (trace, _) = test_drive(algorithm, Context::default(), request(), echo()).await?;let decision = &trace[0];assert_eq!(decision.selected_model(), "only/model");diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex b7a370e5a..a244b4bbd 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -231,7 +231,7 @@ mod tests {use crate::core::algorithm::LlmTarget;use crate::core::classifier::Score;use crate::core::state::StateValue;- use crate::core::testing::{Serve, drive, reply};+ use crate::core::testing::{Serve, reply, test_drive};use switchyard_protocol::{Context, Decision, Metadata, Response};fn tier_target(name: &str) -> LlmTarget {@@ -494,14 +494,14 @@ mod tests {let router = recording_router(config_with_notes())?;let ctx = Context::default();- drive(+ test_drive(router.clone(),ctx.clone(),turn_request(false),recorder.serve(),).await?;- drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;+ test_drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;let calls = recorder.routed();assert_eq!(calls[0].target, "weak");@@ -527,7 +527,7 @@ mod tests {let recorder = Arc::new(Recorder::default());let router = recording_router(config_with_judge(&recorder, 0.1))?;- drive(+ test_drive(router.clone(),Context::default(),turn_request(false),@@ -548,7 +548,7 @@ mod tests {let recorder = Arc::new(Recorder::default());let router = recording_router(config_with_judge(&recorder, 0.9))?;- drive(+ test_drive(router.clone(),Context::default(),turn_request(true),@@ -570,7 +570,7 @@ mod tests {let router = recording_router(config_with_judge(&recorder, 0.1))?;let ctx = Context::default();- drive(+ test_drive(router.clone(),ctx.clone(),turn_request(false),@@ -578,7 +578,7 @@ mod tests {).await?;*recorder.judge_p_solve.lock() = 0.9;- drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;+ test_drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;let routed = recorder.routed();assert_eq!(routed[0].target, "strong");@@ -601,7 +601,7 @@ mod tests {let recorder = Arc::new(Recorder::default());let router = recording_router(config_with_judge(&recorder, 42.0))?;- drive(+ test_drive(router.clone(),Context::default(),turn_request(false),@@ -618,7 +618,7 @@ mod tests {let recorder = Arc::new(Recorder::default());let router = recording_router(config_with_judge(&recorder, 0.9))?;- drive(+ test_drive(router.clone(),Context::default(),turn_request(false),diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rsindex 87ef0fcdc..6fde55d14 100644--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs@@ -17,7 +17,7 @@ use super::util::subagent::SubagentOverride;use crate::Result;use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet};use crate::core::classifier::{Classification, Classifier, Score};-use crate::core::testing::{drive, echo};+use crate::core::testing::{echo, test_drive};use switchyard_protocol::{Context, Metadata, Request, Response, completion_text, slice_to_header_map, text_request,};@@ -77,7 +77,8 @@ fn router() -> Arc<FallThrough> {/// Runs one turn, returning the target that served it.async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {- let (_, response) = drive(router.clone(), Context::default(), request(headers), echo()).await?;+ let (_, response) =+ test_drive(router.clone(), Context::default(), request(headers), echo()).await?;Ok(response.llm_response.as_agg()diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex d7179eb24..463ebd647 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -10,7 +10,7 @@ use std::{future::Future,pin::Pin,sync::Arc,- time::{Duration, Instant},+ time::Instant,};use async_trait::async_trait;@@ -26,7 +26,7 @@ use tracing::Instrument;/// [`switchyard_protocol::LlmResponse`] carries either a live/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.use switchyard_protocol::{- Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals, Usage,+ Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals,};use super::driver::{DriverRequest, DriverStep, TypeErasedDriver};@@ -37,39 +37,6 @@ use crate::{DriverError, LibsyError, Result, observability};/// `Arc<dyn Algorithm>` object-safe.pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;-/// One completed model call observed at the algorithm offload boundary.-#[derive(Clone, Debug)]-pub struct LlmCallObservation {- /// Model selected for the completed call.- pub selected_model: String,- /// Routing tier attached to the selected model, when present.- pub tier: Option<String>,- /// Whether this was the routed backend call rather than classifier or judge overhead.- pub is_routed: bool,- /// Whether the call completed successfully.- pub is_success: bool,- /// Time spent waiting for the model call to resolve.- pub duration: Duration,- /// Normalized usage for a buffered successful response.- pub usage: Option<Usage>,-}--/// One request-scoped observation emitted by the algorithm runner.-#[derive(Clone, Debug)]-pub enum RunObservation {- /// A completed model call.- LlmCall(LlmCallObservation),- /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.- RoutingOverhead(Duration),-}--/// Request-scoped callback for algorithm-run observations.-///-/// The runner invokes this callback inline while resolving observations. Several-/// runs may invoke the same observer concurrently, so implementations must be-/// thread-safe, fast, and non-blocking.-pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;-/// A request paired with the routing [`Decision`] that produced it — the offload/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.///@@ -148,36 +115,14 @@ impl CallLlmRequest {#[derive(Clone)]pub struct Driver {driver: TypeErasedDriver,- // How long the call that served this run took. We need this to calculate routing overhead.- routed_call: Arc<Mutex<Option<Duration>>>,- observer: Option<RunObserver>,}impl Driver {/// Build an empty driver with its step channel ready. Created per call by/// [`run_stream`](Algorithm::run_stream).pub(crate) fn new() -> Self {- Self::with_observer(None)- }-- fn with_observer(observer: Option<RunObserver>) -> Self {Self {driver: TypeErasedDriver::new(),- routed_call: Arc::new(Mutex::new(None)),- observer,- }- }-- /// How long the call that served this run took, if one has succeeded.- pub(crate) fn routed_call_duration(&self) -> Option<Duration> {- *self.routed_call.lock()- }-- /// Records routing overhead (how long Switchyard added to the call) once- /// per successful run. Called by `observe_run`.- pub(crate) fn observe_routing_overhead(&self, duration: Duration) {- if let Some(observer) = &self.observer {- observer(RunObservation::RoutingOverhead(duration));}}@@ -225,25 +170,6 @@ impl Driver {&result,&tracing::Span::current(),);- if let Some(observer) = &self.observer {- observer(RunObservation::LlmCall(LlmCallObservation {- selected_model,- tier,- is_routed,- is_success: result.is_ok(),- duration: elapsed,- usage: result- .as_ref()- .ok()- .and_then(|response| response.llm_response.as_agg())- .map(|response| response.usage.clone()),- }));- }- // Classifier and judge calls are routing overhead.- // And don't record time for failed calls.- if is_routed && result.is_ok() {- *self.routed_call.lock() = Some(elapsed);- }result}@@ -315,8 +241,6 @@ pub enum Step {/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.////// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published.-/// `observer`, when present, is passed through to `run_stream`.-////// `serve` owns the call: it performs it however the host likes and must fulfill the promise/// with [`CallLlmRequest::respond`]. A failed *model* call belongs in `respond` — the/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so@@ -330,14 +254,13 @@ pub async fn drive<F, Fut>(algorithm: Arc<dyn Algorithm>,ctx: Context,request: Request,- observer: Option<RunObserver>,serve: F,) -> Result<(Vec<Arc<dyn Decision>>, Response)>whereF: Fn(CallLlmRequest) -> Fut,Fut: Future<Output = Result<()>>,{- let stream = algorithm.run_stream(ctx, request, observer);+ let stream = algorithm.run_stream(ctx, request);tokio::pin!(stream);let mut trace: Vec<Arc<dyn Decision>> = Vec::new();@@ -677,14 +600,8 @@ pub trait Algorithm: Send + Sync + 'static {/// not polling. A successful run ends with [`Step::ReturnToAgent`]; a failure is/// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.///- /// Every invocation owns a separate [`Driver`]. `observer`, when present, receives- /// each completed model call and, after a successful routed run, its routing overhead.- fn run_stream(- self: Arc<Self>,- ctx: Context,- request: Request,- observer: Option<RunObserver>,- ) -> StepStream {+ /// Every invocation owns a separate [`Driver`].+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {// Stamp the algorithm's telemetry label into the request context; the// context rides on every driver call, so its telemetry is attributed.let mut ctx = ctx;@@ -692,7 +609,7 @@ pub trait Algorithm: Send + Sync + 'static {observability::ALGORITHM_KEY.to_string(),self.name().to_string(),);- let driver = Driver::with_observer(observer);+ let driver = Driver::new();let task_driver = driver.clone();let task_ctx = ctx.clone();let stream = task_driver.stream();@@ -700,12 +617,10 @@ pub trait Algorithm: Send + Sync + 'static {// `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s// contextual parenting.let span = observability::run_span(self.name(), &request);- let observed_driver = task_driver.clone();let handle = tokio::spawn(async move {observability::observe_run(task_ctx.clone(),- observed_driver,self.create_run_task(task_ctx, task_driver, request),).await@@ -740,7 +655,7 @@ pub trait Algorithm: Send + Sync + 'static {#[cfg(test)]mod tests {use super::*;- use crate::core::testing::{Serve, ServeResult, drive, drive_observed, echo, reply};+ use crate::core::testing::{Serve, ServeResult, echo, reply, test_drive};use futures::StreamExt;use switchyard_protocol::{LlmResponse, LlmResponseChunk, completion_text, text_request, text_response,@@ -886,39 +801,6 @@ mod tests {LlmTargetSet::new(targets)}- #[tokio::test]- async fn observed_run_reports_one_successful_routed_call() -> Result<()> {- let observations = Arc::new(Mutex::new(Vec::new()));- let observed = observations.clone();- let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));- let (_, response) = drive_observed(- orch(target_set(&["direct/model"])),- Context::default(),- request(),- Some(observer),- echo(),- )- .await?;- assert_eq!(- response.llm_response.as_agg().map(completion_text),- Some("direct/model".to_string())- );- let observations = observations.lock();- assert_eq!(observations.len(), 2);- let RunObservation::LlmCall(observation) = &observations[0] else {- return Err(test_error("expected an LLM call observation"));- };- assert_eq!(observation.selected_model, "direct/model");- assert!(observation.is_routed);- assert!(observation.is_success);- assert!(observation.usage.is_some());- assert!(matches!(- observations[1],- RunObservation::RoutingOverhead(_)- ));- Ok(())- }-#[test]fn target_lookup_returns_the_missing_target() {let error = target_set(&[]).get_target("missing").err();@@ -967,7 +849,7 @@ mod tests {reason: Some("stop".to_string()),},]);- let (trace, response) = drive(orch, Context::default(), request(), serve).await?;+ let (trace, response) = test_drive(orch, Context::default(), request(), serve).await?;// The run handed back the live stream; the caller folds it to a buffered aggregate.let agg = response.llm_response@@ -993,7 +875,7 @@ mod tests {message: "upstream exploded".to_string(),},]);- let (_, response) = drive(orch, Context::default(), request(), serve).await?;+ let (_, response) = test_drive(orch, Context::default(), request(), serve).await?;match response.llm_response.into_agg().await {Ok(_) => panic!("expected a mid-stream error, got an aggregate"),Err(err) => {@@ -1007,8 +889,7 @@ mod tests {async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {// Every call is offloaded via a promise the orchestrator surfaces as a// `CallLlm` step for us to fulfill.- let stream =- orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);+ let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());tokio::pin!(stream);let mut saw_call = false;@@ -1053,7 +934,7 @@ mod tests {#[tokio::test]async fn a_driven_run_returns_the_trace_and_the_final_response() -> Result<()> {- let (trace, response) = drive(+ let (trace, response) = test_drive(orch(target_set(&["direct/model"])),Context::default(),request(),@@ -1100,7 +981,7 @@ mod tests {}};handles.push(tokio::spawn(async move {- drive(algo, Context::default(), request(), serve)+ test_drive(algo, Context::default(), request(), serve).await.map(|(_, response)| {response@@ -1128,8 +1009,7 @@ mod tests {// A client-less target offloads its call; we fulfill the promise with an// Err, which must flow back through `call_llm_target` into the algorithm and// out as an error step — not a response.- let stream =- orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);+ let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());tokio::pin!(stream);let mut saw_error = false;@@ -1203,7 +1083,7 @@ mod tests {dropped: dropped.clone(),});- let stream = algo.run_stream(Context::default(), request(), None);+ let stream = algo.run_stream(Context::default(), request());started_rx.recv().await@@ -1241,7 +1121,7 @@ mod tests {}let algo: Arc<dyn Algorithm> = Arc::new(Panicky);- let stream = algo.run_stream(Context::default(), request(), None);+ let stream = algo.run_stream(Context::default(), request());tokio::pin!(stream);let mut saw_error = false;@@ -1282,7 +1162,7 @@ mod tests {}let algo: Arc<dyn Algorithm> = Arc::new(Panicky);- match drive(algo, Context::default(), request(), echo()).await {+ match test_drive(algo, Context::default(), request(), echo()).await {Ok(_) => Err(test_error("expected the run to surface the algorithm panic as an error",)),@@ -1344,7 +1224,9 @@ mod tests {// Drive the run on its own task, wait until the algorithm task is up, then cancel// it — dropping its future (and the `run_stream` stream it holds).let run_task =- tokio::spawn(async move { drive(algo, Context::default(), request(), echo()).await });+ tokio::spawn(+ async move { test_drive(algo, Context::default(), request(), echo()).await },+ );started_rx.recv().await@@ -1440,7 +1322,7 @@ mod tests {// The loser responds 50ms after the winner has already won. `run` must return the// winner, not the loser's `respond`-to-a-dropped-receiver error.let (algo, serve) = hedge(Some(std::time::Duration::from_millis(50)));- let (_trace, response) = drive(algo, Context::default(), request(), serve).await?;+ let (_trace, response) = test_drive(algo, Context::default(), request(), serve).await?;assert_eq!(response.llm_response@@ -1457,7 +1339,7 @@ mod tests {// The loser never resolves. `run` must return the winner promptly, not hang// waiting for the in-flight loser.let (algo, serve) = hedge(None);- let run = drive(algo, Context::default(), request(), serve);+ let run = test_drive(algo, Context::default(), request(), serve);let (_trace, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run).await.map_err(|error| LibsyError::external("waiting for pending loser", error))??;@@ -1539,7 +1421,7 @@ mod tests {// With the cap gone, the driver keeps polling the stream even with N calls in// flight, so the terminal error surfaces promptly instead of hanging.- let run = drive(algo, Context::default(), request(), serve);+ let run = test_drive(algo, Context::default(), request(), serve);let result = tokio::time::timeout(std::time::Duration::from_millis(500), run).await.map_err(|error| {diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rsindex 9a0e4f7aa..755b69474 100644--- a/crates/libsy/src/core/testing.rs+++ b/crates/libsy/src/core/testing.rs@@ -20,7 +20,7 @@ use switchyard_protocol::{Context, Decision, LlmClientError, LlmResponse, Request, Response, text_response,};-use crate::core::algorithm::{Algorithm, CallLlmRequest, RunObserver};+use crate::core::algorithm::{Algorithm, CallLlmRequest};use crate::{LibsyError, Result};/// The result a fake client hands back for one offloaded call.@@ -51,25 +51,14 @@ where}/// Run `algorithm` to completion, serving each offloaded call with `serve`.-pub(crate) async fn drive(+pub(crate) async fn test_drive(algorithm: Arc<dyn Algorithm>,ctx: Context,request: Request,serve: impl Serve,-) -> Result<(Vec<Arc<dyn Decision>>, Response)> {- drive_observed(algorithm, ctx, request, None, serve).await-}--/// [`drive`] plus a [`RunObserver`], for tests that assert on run observations.-pub(crate) async fn drive_observed(- algorithm: Arc<dyn Algorithm>,- ctx: Context,- request: Request,- observer: Option<RunObserver>,- serve: impl Serve,) -> Result<(Vec<Arc<dyn Decision>>, Response)> {let serve = Arc::new(serve);- crate::drive(algorithm, ctx, request, observer, move |call| {+ crate::drive(algorithm, ctx, request, move |call| {fulfill(Arc::clone(&serve), call)}).awaitdiff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex d6593d23f..fdbcd5e0d 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -6,8 +6,8 @@mod core;pub use core::algorithm::{- Algorithm, CallLlmRequest, Driver, LlmCallObservation, LlmTarget, LlmTargetSet, RoutedRequest,- RunObservation, RunObserver, Step, StepStream, drive,+ Algorithm, CallLlmRequest, Driver, LlmTarget, LlmTargetSet, RoutedRequest, Step, StepStream,+ drive,};pub use core::classifier::{Classification, Classifier, Score};pub use core::processor::{Event, Processor};diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 1e02bcc88..272a2e5fd 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -41,7 +41,7 @@ use opentelemetry::metrics::{Meter, ObservableGauge};use opentelemetry::{KeyValue, global};use tracing::Span;-use crate::{Driver, Result};+use crate::Result;use switchyard_protocol::{Context, Decision, Request, Response};const METRICS_SCOPE: &str = "switchyard";@@ -146,12 +146,10 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {}/// Runs one algorithm task to completion, recording the run counter, duration-/// histogram, routing overhead, span outcome, and failure log when it resolves.+/// histogram, span outcome, and failure log when it resolves./// Executes inside the `libsy.run` span its caller instruments the task with.-/// `driver` is the run's own, holding the duration of the call that served it.pub(crate) async fn observe_run(ctx: Context,- driver: Driver,run: impl Future<Output = Result<Response>>,) -> Result<Response> {let started = Instant::now();@@ -159,12 +157,6 @@ pub(crate) async fn observe_run(let duration = started.elapsed();let algorithm = algorithm_label(&ctx);record_run(algorithm, duration, &result, &Span::current());- if result.is_ok()- && let Some(overhead) =- record_routing_overhead(algorithm, duration, driver.routed_call_duration())- {- driver.observe_routing_overhead(overhead);- }result}@@ -199,28 +191,6 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp.record(duration.as_secs_f64() * 1000.0, &attributes);}-/// Records what routing cost on top of the call that served the run: classifier-/// calls, target resolution, decision publishing. A run with no routed call has-/// nothing to subtract, so it records nothing.-fn record_routing_overhead(- algorithm: &str,- run: Duration,- routed_call: Option<Duration>,-) -> Option<Duration> {- let routed_call = routed_call?;- // Saturating: the two clocks start a moment apart, so a run that is all- // routed call can come out fractionally negative.- let overhead = run.saturating_sub(routed_call);- meter()- .f64_histogram("switchyard.routing_overhead_ms")- .build()- .record(- overhead.as_secs_f64() * 1000.0,- &[KeyValue::new("algorithm", algorithm.to_string())],- );- Some(overhead)-}-/// Records a judge failure that made the classifier route without a verdict.pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static str) {meter()diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 4e4b28993..2c9414687 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -32,11 +32,11 @@ use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Extension, Json, Router};use axum_server::tls_rustls::RustlsConfig;-use libsy::{Algorithm, LibsyError, RunObservation, RunObserver};+use libsy::{Algorithm, LibsyError};use parking_lot::Mutex;use serde::Deserialize;use serde_json::{Value, json};-use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};+use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};use switchyard_protocol::{Context, Decision, LlmClientError, Metadata, Request, Usage};use tokio::net::{TcpListener, TcpSocket};use tokio::task;@@ -689,16 +689,14 @@ async fn handle_llm_request(Err(response) => return response,};let algorithm = Arc::clone(&route.algorithm);- // Not one client for the route: a router that resolves each offloaded call to the- // client configured for the target the algorithm selected.- let target_clients = route.target_clients.clone();+ let client_router = route.target_clients.clone();let observer = stats_observer(state.stats.clone(),state.routing_log.clone().zip(routing_log_context.clone()),);let (trace, response) = match switchyard_llm_client::run(algorithm,- target_clients,+ client_router,Context::default(),request,Some(observer),@@ -1318,7 +1316,7 @@ fn endpoint_listing(has_routing_log: bool) -> String {#[cfg(test)]mod tests {- use libsy::LlmCallObservation;+ use switchyard_llm_client::LlmCallObservation;use tokio::io::{AsyncReadExt, AsyncWriteExt};use tokio::sync::{Notify, oneshot};diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.mdindex 9185b047b..6487d495f 100644--- a/docs/internal/metrics_reference.md+++ b/docs/internal/metrics_reference.md@@ -57,7 +57,7 @@ Each histogram emits `_bucket`, `_sum`, and `_count` series. Use| Metric | Type | Meaning ||---|---|---|-| `switchyard_routing_overhead_ms{algorithm}` | histogram | Algorithm run time minus the final routed model call. Includes classifier calls, target resolution, and decision publication; runs with no final routed call are not recorded. |+| `switchyard_routing_overhead_ms{algorithm}` | histogram | Total run time minus the time spent in successful routed model calls, with overlapping hedged calls counted once. Includes classifier calls, failed routed attempts, target resolution, and decision publication; runs with no successful routed call are not recorded. Measured across the whole run, so it does not reconcile with `switchyard_run_duration_ms`, which times only the algorithm task. |## Classifier fail-open counter@@ -160,6 +160,6 @@ into label space.|---|---|| `model="<unknown>"` rows appear | A routed-call observation did not include a selected model. || All counters at 0 after warm-up | Server just started with no traffic, or the scraper is hitting the wrong port. |-| `switchyard_routing_overhead_ms_count` stuck at `0` | No successful algorithm run has recorded a final routed model call. |+| `switchyard_routing_overhead_ms_count` stuck at `0` | No successful algorithm run has recorded a successful routed model call. || `switchyard_classifier_fail_open_total` rising | The judge target is failing or returning a response the classifier cannot parse. Check `judge_model` and `reason`. || `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky, or retries are exhausting; compare client responses with retryable upstream attempts. |