Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat: Move observation from libsy to libsy-llm-client (#332)

Graham Kinge8390d43e88c
diff --git a/crates/libsy-llm-client/Cargo.toml b/crates/libsy-llm-client/Cargo.toml
index 0bba91472..9a9ca6e0a 100644
--- a/crates/libsy-llm-client/Cargo.toml
+++ b/crates/libsy-llm-client/Cargo.toml
@@ -25,6 +25,7 @@ async-trait.workspace = true
futures.workspace = true
futures-util.workspace = true
opentelemetry = { version = "0.32", default-features = false, features = ["metrics"] }
+parking_lot.workspace = true
http.workspace = true
httpdate.workspace = true
serde_json.workspace = true
@@ -35,7 +36,6 @@ tracing-opentelemetry.workspace = true
[dev-dependencies]
# SDK + in-memory exporter to assert what the observability layer records.
opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
-parking_lot.workspace = true
thiserror.workspace = true
tracing-subscriber.workspace = true
wiremock = "0.6"
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index f64259c3d..54c9737b6 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -23,7 +23,7 @@ use tracing::Instrument;
use crate::backend::Backend;
use crate::error::{LlmClientError, Result};
-use crate::metrics::{is_retryable_http_status, record_upstream_attempt};
+use crate::metrics;
use crate::raw::RawResponse;
// TODO: Why is this here? What does it do?
@@ -287,7 +287,7 @@ impl TranslatingLlmClient {
let response = match builder.send().await {
Ok(response) => response,
Err(error) => {
- record_upstream_attempt(None);
+ metrics::record_upstream_attempt(None);
return Err(AttemptFailure {
error: convert_reqwest_error(error),
status: None,
@@ -299,13 +299,13 @@ impl TranslatingLlmClient {
if status.is_success() {
if streaming {
// Streaming body failures happen after the retry boundary.
- record_upstream_attempt(Some(status.as_u16()));
+ metrics::record_upstream_attempt(Some(status.as_u16()));
return Ok(EncodedResponse::Streaming(response));
}
let body = match response.bytes().await {
Ok(body) => body,
Err(error) => {
- record_upstream_attempt(None);
+ metrics::record_upstream_attempt(None);
return Err(AttemptFailure {
error: convert_reqwest_error(error),
status: Some(status.as_u16()),
@@ -313,7 +313,7 @@ impl TranslatingLlmClient {
});
}
};
- record_upstream_attempt(Some(status.as_u16()));
+ metrics::record_upstream_attempt(Some(status.as_u16()));
return Ok(EncodedResponse::Buffered {
status: status.as_u16(),
body: body.to_vec(),
@@ -324,7 +324,7 @@ impl TranslatingLlmClient {
let body = match response.text().await {
Ok(body) => body,
Err(error) => {
- record_upstream_attempt(None);
+ metrics::record_upstream_attempt(None);
return Err(AttemptFailure {
error: convert_reqwest_error(error),
status: Some(status.as_u16()),
@@ -332,7 +332,7 @@ impl TranslatingLlmClient {
});
}
};
- record_upstream_attempt(Some(status.as_u16()));
+ metrics::record_upstream_attempt(Some(status.as_u16()));
let error =
if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) {
LlmClientError::ContextWindowExceeded {
@@ -563,7 +563,9 @@ impl AttemptFailure {
fn is_retryable(&self) -> bool {
match &self.error {
LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true,
- LlmClientError::UpstreamHttp { status, .. } => is_retryable_http_status(*status),
+ LlmClientError::UpstreamHttp { status, .. } => {
+ metrics::is_retryable_http_status(*status)
+ }
_ => false,
}
}
diff --git a/crates/libsy-llm-client/src/lib.rs b/crates/libsy-llm-client/src/lib.rs
index f1c8a462d..e10a17dd6 100644
--- a/crates/libsy-llm-client/src/lib.rs
+++ b/crates/libsy-llm-client/src/lib.rs
@@ -21,12 +21,14 @@ pub mod client;
pub mod error;
pub mod metrics;
mod observability;
+mod observation;
pub mod raw;
pub mod run;
pub use backend::{Backend, DEFAULT_MAX_RETRIES, HttpBackendConfig};
pub use client::{ModelConfig, TranslatingLlmClient};
pub use error::{LlmClientError, Result};
+pub use observation::{LlmCallObservation, RunObservation, RunObserver};
pub use raw::RawResponse;
pub use run::{ClientRouter, run};
pub use switchyard_translation::RawEventStream;
diff --git a/crates/libsy-llm-client/src/metrics.rs b/crates/libsy-llm-client/src/metrics.rs
index 672d8dd5f..582bb5f6b 100644
--- a/crates/libsy-llm-client/src/metrics.rs
+++ b/crates/libsy-llm-client/src/metrics.rs
@@ -3,6 +3,8 @@
//! Metric labelling inherited from Python
+use std::time::Duration;
+
use opentelemetry::{KeyValue, global};
pub(crate) const fn is_retryable_http_status(status: u16) -> bool {
@@ -59,6 +61,26 @@ pub(crate) fn record_upstream_attempt(status: Option<u16>) {
);
}
+/// Records what routing cost on top of the call that served the run: classifier
+/// calls, target resolution, and decision publishing.
+pub(crate) fn record_routing_overhead(
+ algorithm: &str,
+ run: Duration,
+ call_duration: Duration,
+) -> Duration {
+ // Saturating: the two clocks start a moment apart, so a run that is all
+ // routed call can come out fractionally negative.
+ let overhead = run.saturating_sub(call_duration);
+ global::meter("switchyard")
+ .f64_histogram("switchyard.routing_overhead_ms")
+ .build()
+ .record(
+ overhead.as_secs_f64() * 1000.0,
+ &[KeyValue::new("algorithm", algorithm.to_string())],
+ );
+ overhead
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs
index 1767ffad8..58270cc03 100644
--- a/crates/libsy-llm-client/src/observability.rs
+++ b/crates/libsy-llm-client/src/observability.rs
@@ -1,18 +1,6 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! `tracing` span fields for the provider call [`run`](crate::run) makes.
-//!
-//! libsy instruments the orchestration layer (`libsy.run`, `libsy.llm_call`) and owns the
-//! `switchyard.*` metrics. This module owns the other half: the `libsy.client_call` span that
-//! covers the one HTTP call a routed request performs, recorded as OpenTelemetry gen_ai
-//! semantic-convention attributes.
-//!
-//! Nothing here records a metric — these are span fields only, written through the `tracing`
-//! facade so the host's subscriber decides where they go. A streamed response resolves before
-//! its usage is known, so [`observe_client_call`] wraps the stream and keeps the span alive
-//! until it drains, errors, or is abandoned.
-
use std::borrow::Cow;
use std::pin::Pin;
use std::task::{Context as TaskContext, Poll};
diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rs
new file mode 100644
index 000000000..c6b955081
--- /dev/null
+++ b/crates/libsy-llm-client/src/observation.rs
@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Request-scoped observations emitted while serving an algorithm run.
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use switchyard_protocol::Usage;
+
+/// One completed model call observed at the algorithm offload boundary.
+#[derive(Clone, Debug)]
+pub struct LlmCallObservation {
+ /// Model selected for the completed call.
+ pub selected_model: String,
+ /// Routing tier attached to the selected model, when present.
+ pub tier: Option<String>,
+ /// Whether this was the routed backend call rather than classifier or judge overhead.
+ pub is_routed: bool,
+ /// Whether the call completed successfully.
+ pub is_success: bool,
+ /// Time spent waiting for the model call to resolve.
+ pub duration: Duration,
+ /// Normalized usage for a buffered successful response.
+ pub usage: Option<Usage>,
+}
+
+/// One request-scoped observation emitted by the algorithm runner.
+#[derive(Clone, Debug)]
+pub enum RunObservation {
+ /// A completed model call.
+ LlmCall(LlmCallObservation),
+ /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.
+ RoutingOverhead(Duration),
+}
+
+/// Request-scoped callback for algorithm-run observations.
+///
+/// The runner invokes this callback inline while resolving observations. Several
+/// runs may invoke the same observer concurrently, so implementations must be
+/// thread-safe, fast, and non-blocking.
+pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index efa500ab0..c3655a07b 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -13,13 +13,14 @@
use std::collections::HashMap;
use std::sync::Arc;
+use std::time::{Duration, Instant};
-use switchyard_libsy::{
- Algorithm, CallLlmRequest, LibsyError, Result, RunObserver, algorithm_label, drive,
-};
+use parking_lot::Mutex;
+use switchyard_libsy::{Algorithm, CallLlmRequest, LibsyError, Result, algorithm_label, drive};
use switchyard_protocol::{Context, Decision, LlmClientError, Request, Response, RoutedLlmClient};
-use crate::observability;
+use crate::observation::{LlmCallObservation, RunObservation, RunObserver};
+use crate::{metrics, observability};
/// Run one request to completion, serving every offloaded model call with `client`.
///
@@ -41,10 +42,61 @@ pub async fn run(
request: Request,
observer: Option<RunObserver>,
) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
- drive(algorithm, ctx, request, observer, move |call| {
- serve(clients.clone(), call)
+ let algorithm_name = algorithm.name().to_string();
+ // The output from `serve` goes in here: when each successful routed call was in
+ // flight. Everything else the run spent time on is routing overhead.
+ let routed_calls = Arc::new(Mutex::new(RoutedCallWindows::default()));
+ let run_started = Instant::now();
+ let result = drive(algorithm, ctx, request, {
+ let observer = observer.clone();
+ let routed_calls = Arc::clone(&routed_calls);
+ move |call| {
+ serve(
+ clients.clone(),
+ call,
+ observer.clone(),
+ Arc::clone(&routed_calls),
+ )
+ }
})
- .await
+ .await?;
+ if let Some(served) = routed_calls.lock().served() {
+ let overhead =
+ metrics::record_routing_overhead(&algorithm_name, run_started.elapsed(), served);
+ if let Some(observer) = observer {
+ observer(RunObservation::RoutingOverhead(overhead));
+ }
+ }
+ Ok(result)
+}
+
+/// The wall-clock windows during which a successful routed call was in flight.
+/// An algorithm can make multiple overlapping calls. This is the union of all of them.
+#[derive(Default)]
+struct RoutedCallWindows(Vec<(Instant, Instant)>);
+
+impl RoutedCallWindows {
+ /// Record one completed routed call.
+ fn record(&mut self, started: Instant, ended: Instant) {
+ self.0.push((started, ended));
+ }
+
+ /// Total time at least one routed call was in flight, merging overlapping windows.
+ fn served(&mut self) -> Option<Duration> {
+ self.0.sort_unstable_by_key(|(started, _)| *started);
+ // Sweep the windows in start order, advancing a cursor along the timeline and
+ // counting only the time each window covers that the cursor has not reached.
+ let mut covered = self.0.first()?.0;
+ let mut total = Duration::ZERO;
+ for &(started, ended) in &self.0 {
+ covered = covered.max(started);
+ if ended > covered {
+ total += ended - covered;
+ covered = ended;
+ }
+ }
+ Some(total)
+ }
}
/// Serve one offloaded call. A failed *model* call is forwarded to the algorithm via
@@ -88,7 +140,13 @@ pub async fn run(
error = tracing::field::Empty,
)
)]
-async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {
+async fn serve(
+ clients: ClientRouter,
+ call: CallLlmRequest,
+ observer: Option<RunObserver>,
+ // Output parameter because `drive` takes a function that returns a plain `Result<()>`.
+ routed_calls: Arc<Mutex<RoutedCallWindows>>,
+) -> Result<()> {
let span = tracing::Span::current();
observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
if let Some(tier) = call.get_decision().routing_tier() {
@@ -105,7 +163,13 @@ async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {
}
let routed = call.get_routed().clone();
let target = routed.decision.selected_model().to_string();
- let result = match clients.route(&target) {
+ let tier = call.get_decision().routing_tier().map(str::to_string);
+ let is_routed = call.get_decision().is_routed_call();
+ // Resolved before the clock starts: picking the client is Switchyard's work, not
+ // the provider's, so it belongs in the routing overhead.
+ let client = clients.route(&target);
+ let started = Instant::now();
+ let result = match client {
Ok(client) => {
client
.call(routed.ctx, routed.request, routed.decision)
@@ -114,7 +178,28 @@ async fn serve(clients: ClientRouter, call: CallLlmRequest) -> Result<()> {
Err(error) => Err(error),
}
.map_err(|source| LibsyError::client_call(target, source));
+ let ended = Instant::now();
+ let duration = ended - started;
+
let result = observability::observe_client_call(result);
+ if let Some(observer) = observer {
+ observer(RunObservation::LlmCall(LlmCallObservation {
+ selected_model: call.get_decision().selected_model().to_string(),
+ tier,
+ is_routed,
+ is_success: result.is_ok(),
+ duration,
+ usage: result
+ .as_ref()
+ .ok()
+ .and_then(|response| response.llm_response.as_agg())
+ .map(|response| response.usage.clone()),
+ }));
+ }
+ if is_routed && result.is_ok() {
+ routed_calls.lock().record(started, ended);
+ }
+
call.respond(result)
}
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 864c95f56..721b722d0 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -36,7 +36,7 @@ use switchyard_libsy::{
Algorithm, Driver, LibsyError, LlmClassifierConfig, LlmTarget, LlmTargetSet, LlmTaskClassifier,
RoutedRequest, Step, TaskClassifierConfig,
};
-use switchyard_llm_client::ClientRouter;
+use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
use switchyard_protocol::{
Context, Decision, LlmResponse, Metadata, Request, Response, RoutedLlmClient, Usage,
};
@@ -843,6 +843,50 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
Ok(())
}
+#[tokio::test]
+async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
+ let observations = Arc::new(Mutex::new(Vec::new()));
+ let observed = Arc::clone(&observations);
+ let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
+ const ALGO: &str = "observed-run-algo";
+ const MODEL: &str = "observed-run-model";
+ let client = Arc::new(UsageClient {
+ usage: Usage::default(),
+ }) as Arc<dyn RoutedLlmClient>;
+
+ let (_, response) = switchyard_llm_client::run(
+ algo(ALGO, MODEL),
+ ClientRouter::single(client),
+ Context::default(),
+ request_with_metadata("observed-session", "observed-correlation"),
+ Some(observer),
+ )
+ .await?;
+
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(|response| response.model.as_deref()),
+ Some(Some(MODEL))
+ );
+ let observations = observations.lock();
+ assert_eq!(observations.len(), 2);
+ let RunObservation::LlmCall(observation) = &observations[0] else {
+ return Err(test_error("expected an LLM call observation"));
+ };
+ assert_eq!(observation.selected_model, MODEL);
+ assert!(observation.is_routed);
+ assert!(observation.is_success);
+ assert!(observation.usage.is_some());
+ assert!(matches!(
+ observations[1],
+ RunObservation::RoutingOverhead(_)
+ ));
+ Ok(())
+}
+
/// A streamed response keeps the client span available until terminal usage arrives.
struct StreamingUsageClient;
@@ -1006,7 +1050,6 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
let stream = algo(ALGO, MODEL).run_stream(
Context::default(),
request_with_metadata("obs-session-2", "obs-corr-2"),
- None,
);
tokio::pin!(stream);
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 331943354..60a7c8808 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -463,7 +463,7 @@ mod tests {
use crate::core::classifier::Classification;
use crate::{AffinityRouter, SystemPromptProcessor, TargetPrompts};
- use crate::core::testing::{Serve, drive, echo, reply};
+ use crate::core::testing::{Serve, echo, reply, test_drive};
use switchyard_protocol::{
LlmClientError, LlmRequest, Message, Metadata, Role, completion_text, text_request,
};
@@ -562,7 +562,7 @@ mod tests {
recorder: &Arc<PromptRecorder>,
router: FallThrough,
) -> Result<RecordedCall> {
- drive(
+ test_drive(
Arc::new(router),
Context::default(),
Request {
@@ -648,7 +648,8 @@ mod tests {
where
S: Default + Send + 'static,
{
- let (trace, response) = drive(router.clone(), Context::default(), request, serve).await?;
+ let (trace, response) =
+ test_drive(router.clone(), Context::default(), request, serve).await?;
let text = response
.llm_response
.into_agg()
@@ -958,7 +959,7 @@ mod tests {
);
let mut ctx = Context::default();
ctx.exclude_target("weak");
- let (trace, response) = drive(router, ctx, request(), echo()).await?;
+ let (trace, response) = test_drive(router, ctx, request(), echo()).await?;
let text = response
.llm_response
.into_agg()
@@ -1252,7 +1253,7 @@ mod tests {
..request()
};
- let result = drive(router.clone(), Context::default(), final_request, echo()).await;
+ let result = test_drive(router.clone(), Context::default(), final_request, echo()).await;
assert!(matches!(result, Err(LibsyError::AlgorithmError { .. })));
let states = router
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index d75748d14..d5be7492f 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -959,7 +959,7 @@ mod tests {
};
use crate::algorithms::util::llm_judge::Judge;
- use crate::core::testing::{Serve, drive, reply};
+ use crate::core::testing::{Serve, reply, test_drive};
use switchyard_protocol::{Context, LlmResponse, Response};
const TEST_THRESHOLD: f64 = 0.5;
@@ -1128,7 +1128,7 @@ mod tests {
async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {
let router = router()?;
- let (trace, response) = drive(
+ let (trace, response) = test_drive(
router,
Context::default(),
classify_request(),
@@ -1150,14 +1150,14 @@ mod tests {
let router = router()?;
let request = classify_request;
- drive(
+ test_drive(
router.clone(),
Context::default(),
request(),
recorder.serve(),
)
.await?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
request(),
@@ -1188,7 +1188,7 @@ mod tests {
},
})?);
- drive(
+ test_drive(
router,
Context::default(),
classify_request(),
@@ -1217,7 +1217,7 @@ mod tests {
},
})?);
- drive(
+ test_drive(
router,
Context::default(),
classify_request(),
@@ -1248,14 +1248,14 @@ mod tests {
})?);
let session_request = classify_session_request;
- drive(
+ test_drive(
router.clone(),
Context::default(),
session_request(),
recorder.serve(),
)
.await?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
session_request(),
@@ -1285,14 +1285,14 @@ mod tests {
},
})?);
- drive(
+ test_drive(
router.clone(),
Context::default(),
classify_request(),
recorder.serve(),
)
.await?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
classify_follow_up_request(),
@@ -1860,7 +1860,7 @@ mod tests {
let model = Queue::new(["efficient answer"]);
let router = escalation_router()?;
- let (trace, response) = drive(
+ let (trace, response) = test_drive(
router,
Context::default(),
classify_request(),
@@ -1895,7 +1895,7 @@ mod tests {
max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS,
})?);
- drive(
+ test_drive(
router,
Context::default(),
classify_request(),
@@ -1917,7 +1917,7 @@ mod tests {
let model = Queue::new(["efficient draft", "capable answer"]);
let router = escalation_router()?;
- let (trace, response) = drive(
+ let (trace, response) = test_drive(
router,
Context::default(),
classify_request(),
@@ -1942,14 +1942,14 @@ mod tests {
let router = escalation_router()?;
let session_request = classify_session_request();
- drive(
+ test_drive(
router.clone(),
Context::default(),
session_request.clone(),
queued(Arc::clone(&model), Arc::clone(&judge)),
)
.await?;
- let (trace, _) = drive(
+ let (trace, _) = test_drive(
router.clone(),
Context::default(),
session_request,
@@ -1981,7 +1981,7 @@ mod tests {
};
let (trace, response) =
- drive(router, Context::default(), classify_request(), serve).await?;
+ test_drive(router, Context::default(), classify_request(), serve).await?;
assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
assert_eq!(
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 66e2b9cdb..853c297a1 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -79,7 +79,7 @@ mod tests {
use switchyard_protocol::{LlmRequest, Message, Role};
use super::*;
- use crate::core::testing::{drive, echo};
+ use crate::core::testing::{echo, test_drive};
#[tokio::test]
async fn test_noop_algo() -> Result<()> {
@@ -97,7 +97,7 @@ mod tests {
// `Noop` synthesizes its own response and never offloads a call, so `echo` is
// never reached.
let a: Arc<dyn Algorithm> = Arc::new(Noop {});
- let (decisions, response) = drive(a, Context::default(), request, echo()).await?;
+ let (decisions, response) = test_drive(a, Context::default(), request, echo()).await?;
let Some(decision) = decisions.first() else {
panic!("Expected exactly one Decision");
};
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 85817ba4d..c8c74faeb 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -73,7 +73,7 @@ mod tests {
use super::Passthrough;
use crate::core::algorithm::{Algorithm, LlmTarget};
- use crate::core::testing::{drive, echo};
+ use crate::core::testing::{echo, test_drive};
use switchyard_protocol::{Context, Request, completion_text, text_request};
#[tokio::test]
@@ -87,7 +87,7 @@ mod tests {
let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {
semantic_name: MODEL_ID.to_string(),
}));
- let (trace, response) = drive(algorithm, Context::default(), request, echo()).await?;
+ let (trace, response) = test_drive(algorithm, Context::default(), request, echo()).await?;
assert_eq!(
response
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index caa3fecc5..a8017cad7 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -182,7 +182,7 @@ mod tests {
use crate::DriverError;
use crate::algorithms::util::affinity::AffinityRouter;
use crate::core::algorithm::LlmTarget;
- use crate::core::testing::{drive, echo};
+ use crate::core::testing::{echo, test_drive};
use switchyard_protocol::{Request, Signals};
fn request() -> Request {
@@ -225,7 +225,7 @@ mod tests {
let mut selected = Vec::with_capacity(count);
for _ in 0..count {
let (_, response) =
- drive(algorithm.clone(), Context::default(), request(), echo()).await?;
+ test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;
selected.push(
response
.llm_response
@@ -240,7 +240,8 @@ mod tests {
#[tokio::test]
async fn single_target_is_always_selected_and_called() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
- let (trace, response) = drive(algorithm, Context::default(), request(), echo()).await?;
+ let (trace, response) =
+ test_drive(algorithm, Context::default(), request(), echo()).await?;
assert_eq!(
response
@@ -262,7 +263,7 @@ mod tests {
for _ in 0..50 {
let (trace, response) =
- drive(algorithm.clone(), Context::default(), request(), echo()).await?;
+ test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;
let selected = response
.llm_response
.as_agg()
@@ -284,7 +285,7 @@ mod tests {
for _ in 0..100 {
let (_, response) =
- drive(algorithm.clone(), Context::default(), request(), echo()).await?;
+ test_drive(algorithm.clone(), Context::default(), request(), echo()).await?;
seen.insert(
response
.llm_response
@@ -348,7 +349,7 @@ mod tests {
.with_classifier(random),
);
- let (_, first) = drive(
+ let (_, first) = test_drive(
algorithm.clone(),
Context::default(),
request_for_session("session-1"),
@@ -373,7 +374,7 @@ mod tests {
Some(selected.to_string())
);
- let (_, second) = drive(
+ let (_, second) = test_drive(
algorithm,
Context::default(),
request_for_session("session-1"),
@@ -431,7 +432,7 @@ mod tests {
#[tokio::test]
async fn decision_is_inspectable_and_downcasts() -> Result<()> {
let algorithm = shared_algorithm(&["only/model"])?;
- let (trace, _) = drive(algorithm, Context::default(), request(), echo()).await?;
+ let (trace, _) = test_drive(algorithm, Context::default(), request(), echo()).await?;
let decision = &trace[0];
assert_eq!(decision.selected_model(), "only/model");
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index b7a370e5a..a244b4bbd 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -231,7 +231,7 @@ mod tests {
use crate::core::algorithm::LlmTarget;
use crate::core::classifier::Score;
use crate::core::state::StateValue;
- use crate::core::testing::{Serve, drive, reply};
+ use crate::core::testing::{Serve, reply, test_drive};
use switchyard_protocol::{Context, Decision, Metadata, Response};
fn tier_target(name: &str) -> LlmTarget {
@@ -494,14 +494,14 @@ mod tests {
let router = recording_router(config_with_notes())?;
let ctx = Context::default();
- drive(
+ test_drive(
router.clone(),
ctx.clone(),
turn_request(false),
recorder.serve(),
)
.await?;
- drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;
+ test_drive(router.clone(), ctx, turn_request(true), recorder.serve()).await?;
let calls = recorder.routed();
assert_eq!(calls[0].target, "weak");
@@ -527,7 +527,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.1))?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
turn_request(false),
@@ -548,7 +548,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.9))?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
turn_request(true),
@@ -570,7 +570,7 @@ mod tests {
let router = recording_router(config_with_judge(&recorder, 0.1))?;
let ctx = Context::default();
- drive(
+ test_drive(
router.clone(),
ctx.clone(),
turn_request(false),
@@ -578,7 +578,7 @@ mod tests {
)
.await?;
*recorder.judge_p_solve.lock() = 0.9;
- drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;
+ test_drive(router.clone(), ctx, turn_request(false), recorder.serve()).await?;
let routed = recorder.routed();
assert_eq!(routed[0].target, "strong");
@@ -601,7 +601,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 42.0))?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
turn_request(false),
@@ -618,7 +618,7 @@ mod tests {
let recorder = Arc::new(Recorder::default());
let router = recording_router(config_with_judge(&recorder, 0.9))?;
- drive(
+ test_drive(
router.clone(),
Context::default(),
turn_request(false),
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index 87ef0fcdc..6fde55d14 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -17,7 +17,7 @@ use super::util::subagent::SubagentOverride;
use crate::Result;
use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet};
use crate::core::classifier::{Classification, Classifier, Score};
-use crate::core::testing::{drive, echo};
+use crate::core::testing::{echo, test_drive};
use switchyard_protocol::{
Context, Metadata, Request, Response, completion_text, slice_to_header_map, text_request,
};
@@ -77,7 +77,8 @@ fn router() -> Arc<FallThrough> {
/// Runs one turn, returning the target that served it.
async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {
- let (_, response) = drive(router.clone(), Context::default(), request(headers), echo()).await?;
+ let (_, response) =
+ test_drive(router.clone(), Context::default(), request(headers), echo()).await?;
Ok(response
.llm_response
.as_agg()
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index d7179eb24..463ebd647 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -10,7 +10,7 @@ use std::{
future::Future,
pin::Pin,
sync::Arc,
- time::{Duration, Instant},
+ time::Instant,
};
use async_trait::async_trait;
@@ -26,7 +26,7 @@ use tracing::Instrument;
/// [`switchyard_protocol::LlmResponse`] carries either a live
/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
use switchyard_protocol::{
- Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals, Usage,
+ Context, Decision, LlmClientError, Request, Response, RoutingFallbackReason, Signals,
};
use super::driver::{DriverRequest, DriverStep, TypeErasedDriver};
@@ -37,39 +37,6 @@ use crate::{DriverError, LibsyError, Result, observability};
/// `Arc<dyn Algorithm>` object-safe.
pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;
-/// One completed model call observed at the algorithm offload boundary.
-#[derive(Clone, Debug)]
-pub struct LlmCallObservation {
- /// Model selected for the completed call.
- pub selected_model: String,
- /// Routing tier attached to the selected model, when present.
- pub tier: Option<String>,
- /// Whether this was the routed backend call rather than classifier or judge overhead.
- pub is_routed: bool,
- /// Whether the call completed successfully.
- pub is_success: bool,
- /// Time spent waiting for the model call to resolve.
- pub duration: Duration,
- /// Normalized usage for a buffered successful response.
- pub usage: Option<Usage>,
-}
-
-/// One request-scoped observation emitted by the algorithm runner.
-#[derive(Clone, Debug)]
-pub enum RunObservation {
- /// A completed model call.
- LlmCall(LlmCallObservation),
- /// Routing time recorded by the `switchyard.routing_overhead_ms` metric.
- RoutingOverhead(Duration),
-}
-
-/// Request-scoped callback for algorithm-run observations.
-///
-/// The runner invokes this callback inline while resolving observations. Several
-/// runs may invoke the same observer concurrently, so implementations must be
-/// thread-safe, fast, and non-blocking.
-pub type RunObserver = Arc<dyn Fn(RunObservation) + Send + Sync>;
-
/// A request paired with the routing [`Decision`] that produced it — the offload
/// payload a host reads (via [`CallLlmRequest::get_routed`]) to serve the call.
///
@@ -148,36 +115,14 @@ impl CallLlmRequest {
#[derive(Clone)]
pub struct Driver {
driver: TypeErasedDriver,
- // How long the call that served this run took. We need this to calculate routing overhead.
- routed_call: Arc<Mutex<Option<Duration>>>,
- observer: Option<RunObserver>,
}
impl Driver {
/// Build an empty driver with its step channel ready. Created per call by
/// [`run_stream`](Algorithm::run_stream).
pub(crate) fn new() -> Self {
- Self::with_observer(None)
- }
-
- fn with_observer(observer: Option<RunObserver>) -> Self {
Self {
driver: TypeErasedDriver::new(),
- routed_call: Arc::new(Mutex::new(None)),
- observer,
- }
- }
-
- /// How long the call that served this run took, if one has succeeded.
- pub(crate) fn routed_call_duration(&self) -> Option<Duration> {
- *self.routed_call.lock()
- }
-
- /// Records routing overhead (how long Switchyard added to the call) once
- /// per successful run. Called by `observe_run`.
- pub(crate) fn observe_routing_overhead(&self, duration: Duration) {
- if let Some(observer) = &self.observer {
- observer(RunObservation::RoutingOverhead(duration));
}
}
@@ -225,25 +170,6 @@ impl Driver {
&result,
&tracing::Span::current(),
);
- if let Some(observer) = &self.observer {
- observer(RunObservation::LlmCall(LlmCallObservation {
- selected_model,
- tier,
- is_routed,
- is_success: result.is_ok(),
- duration: elapsed,
- usage: result
- .as_ref()
- .ok()
- .and_then(|response| response.llm_response.as_agg())
- .map(|response| response.usage.clone()),
- }));
- }
- // Classifier and judge calls are routing overhead.
- // And don't record time for failed calls.
- if is_routed && result.is_ok() {
- *self.routed_call.lock() = Some(elapsed);
- }
result
}
@@ -315,8 +241,6 @@ pub enum Step {
/// Drive [`Algorithm::run_stream`] to completion, handing each offloaded call to `serve`.
///
/// Returns the final [`Response`] and the trace of [`Decision`]s the algorithm published.
-/// `observer`, when present, is passed through to `run_stream`.
-///
/// `serve` owns the call: it performs it however the host likes and must fulfill the promise
/// with [`CallLlmRequest::respond`]. A failed *model* call belongs in `respond` — the
/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so
@@ -330,14 +254,13 @@ pub async fn drive<F, Fut>(
algorithm: Arc<dyn Algorithm>,
ctx: Context,
request: Request,
- observer: Option<RunObserver>,
serve: F,
) -> Result<(Vec<Arc<dyn Decision>>, Response)>
where
F: Fn(CallLlmRequest) -> Fut,
Fut: Future<Output = Result<()>>,
{
- let stream = algorithm.run_stream(ctx, request, observer);
+ let stream = algorithm.run_stream(ctx, request);
tokio::pin!(stream);
let mut trace: Vec<Arc<dyn Decision>> = Vec::new();
@@ -677,14 +600,8 @@ pub trait Algorithm: Send + Sync + 'static {
/// not polling. A successful run ends with [`Step::ReturnToAgent`]; a failure is
/// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.
///
- /// Every invocation owns a separate [`Driver`]. `observer`, when present, receives
- /// each completed model call and, after a successful routed run, its routing overhead.
- fn run_stream(
- self: Arc<Self>,
- ctx: Context,
- request: Request,
- observer: Option<RunObserver>,
- ) -> StepStream {
+ /// Every invocation owns a separate [`Driver`].
+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {
// Stamp the algorithm's telemetry label into the request context; the
// context rides on every driver call, so its telemetry is attributed.
let mut ctx = ctx;
@@ -692,7 +609,7 @@ pub trait Algorithm: Send + Sync + 'static {
observability::ALGORITHM_KEY.to_string(),
self.name().to_string(),
);
- let driver = Driver::with_observer(observer);
+ let driver = Driver::new();
let task_driver = driver.clone();
let task_ctx = ctx.clone();
let stream = task_driver.stream();
@@ -700,12 +617,10 @@ pub trait Algorithm: Send + Sync + 'static {
// `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s
// contextual parenting.
let span = observability::run_span(self.name(), &request);
- let observed_driver = task_driver.clone();
let handle = tokio::spawn(
async move {
observability::observe_run(
task_ctx.clone(),
- observed_driver,
self.create_run_task(task_ctx, task_driver, request),
)
.await
@@ -740,7 +655,7 @@ pub trait Algorithm: Send + Sync + 'static {
#[cfg(test)]
mod tests {
use super::*;
- use crate::core::testing::{Serve, ServeResult, drive, drive_observed, echo, reply};
+ use crate::core::testing::{Serve, ServeResult, echo, reply, test_drive};
use futures::StreamExt;
use switchyard_protocol::{
LlmResponse, LlmResponseChunk, completion_text, text_request, text_response,
@@ -886,39 +801,6 @@ mod tests {
LlmTargetSet::new(targets)
}
- #[tokio::test]
- async fn observed_run_reports_one_successful_routed_call() -> Result<()> {
- let observations = Arc::new(Mutex::new(Vec::new()));
- let observed = observations.clone();
- let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
- let (_, response) = drive_observed(
- orch(target_set(&["direct/model"])),
- Context::default(),
- request(),
- Some(observer),
- echo(),
- )
- .await?;
- assert_eq!(
- response.llm_response.as_agg().map(completion_text),
- Some("direct/model".to_string())
- );
- let observations = observations.lock();
- assert_eq!(observations.len(), 2);
- let RunObservation::LlmCall(observation) = &observations[0] else {
- return Err(test_error("expected an LLM call observation"));
- };
- assert_eq!(observation.selected_model, "direct/model");
- assert!(observation.is_routed);
- assert!(observation.is_success);
- assert!(observation.usage.is_some());
- assert!(matches!(
- observations[1],
- RunObservation::RoutingOverhead(_)
- ));
- Ok(())
- }
-
#[test]
fn target_lookup_returns_the_missing_target() {
let error = target_set(&[]).get_target("missing").err();
@@ -967,7 +849,7 @@ mod tests {
reason: Some("stop".to_string()),
},
]);
- let (trace, response) = drive(orch, Context::default(), request(), serve).await?;
+ let (trace, response) = test_drive(orch, Context::default(), request(), serve).await?;
// The run handed back the live stream; the caller folds it to a buffered aggregate.
let agg = response
.llm_response
@@ -993,7 +875,7 @@ mod tests {
message: "upstream exploded".to_string(),
},
]);
- let (_, response) = drive(orch, Context::default(), request(), serve).await?;
+ let (_, response) = test_drive(orch, Context::default(), request(), serve).await?;
match response.llm_response.into_agg().await {
Ok(_) => panic!("expected a mid-stream error, got an aggregate"),
Err(err) => {
@@ -1007,8 +889,7 @@ mod tests {
async fn run_offloads_via_promise_then_returns_to_agent() -> Result<()> {
// Every call is offloaded via a promise the orchestrator surfaces as a
// `CallLlm` step for us to fulfill.
- let stream =
- orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);
+ let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());
tokio::pin!(stream);
let mut saw_call = false;
@@ -1053,7 +934,7 @@ mod tests {
#[tokio::test]
async fn a_driven_run_returns_the_trace_and_the_final_response() -> Result<()> {
- let (trace, response) = drive(
+ let (trace, response) = test_drive(
orch(target_set(&["direct/model"])),
Context::default(),
request(),
@@ -1100,7 +981,7 @@ mod tests {
}
};
handles.push(tokio::spawn(async move {
- drive(algo, Context::default(), request(), serve)
+ test_drive(algo, Context::default(), request(), serve)
.await
.map(|(_, response)| {
response
@@ -1128,8 +1009,7 @@ mod tests {
// A client-less target offloads its call; we fulfill the promise with an
// Err, which must flow back through `call_llm_target` into the algorithm and
// out as an error step — not a response.
- let stream =
- orch(target_set(&["offload/model"])).run_stream(Context::default(), request(), None);
+ let stream = orch(target_set(&["offload/model"])).run_stream(Context::default(), request());
tokio::pin!(stream);
let mut saw_error = false;
@@ -1203,7 +1083,7 @@ mod tests {
dropped: dropped.clone(),
});
- let stream = algo.run_stream(Context::default(), request(), None);
+ let stream = algo.run_stream(Context::default(), request());
started_rx
.recv()
.await
@@ -1241,7 +1121,7 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- let stream = algo.run_stream(Context::default(), request(), None);
+ let stream = algo.run_stream(Context::default(), request());
tokio::pin!(stream);
let mut saw_error = false;
@@ -1282,7 +1162,7 @@ mod tests {
}
let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- match drive(algo, Context::default(), request(), echo()).await {
+ match test_drive(algo, Context::default(), request(), echo()).await {
Ok(_) => Err(test_error(
"expected the run to surface the algorithm panic as an error",
)),
@@ -1344,7 +1224,9 @@ mod tests {
// Drive the run on its own task, wait until the algorithm task is up, then cancel
// it — dropping its future (and the `run_stream` stream it holds).
let run_task =
- tokio::spawn(async move { drive(algo, Context::default(), request(), echo()).await });
+ tokio::spawn(
+ async move { test_drive(algo, Context::default(), request(), echo()).await },
+ );
started_rx
.recv()
.await
@@ -1440,7 +1322,7 @@ mod tests {
// The loser responds 50ms after the winner has already won. `run` must return the
// winner, not the loser's `respond`-to-a-dropped-receiver error.
let (algo, serve) = hedge(Some(std::time::Duration::from_millis(50)));
- let (_trace, response) = drive(algo, Context::default(), request(), serve).await?;
+ let (_trace, response) = test_drive(algo, Context::default(), request(), serve).await?;
assert_eq!(
response
.llm_response
@@ -1457,7 +1339,7 @@ mod tests {
// The loser never resolves. `run` must return the winner promptly, not hang
// waiting for the in-flight loser.
let (algo, serve) = hedge(None);
- let run = drive(algo, Context::default(), request(), serve);
+ let run = test_drive(algo, Context::default(), request(), serve);
let (_trace, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run)
.await
.map_err(|error| LibsyError::external("waiting for pending loser", error))??;
@@ -1539,7 +1421,7 @@ mod tests {
// With the cap gone, the driver keeps polling the stream even with N calls in
// flight, so the terminal error surfaces promptly instead of hanging.
- let run = drive(algo, Context::default(), request(), serve);
+ let run = test_drive(algo, Context::default(), request(), serve);
let result = tokio::time::timeout(std::time::Duration::from_millis(500), run)
.await
.map_err(|error| {
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index 9a0e4f7aa..755b69474 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -20,7 +20,7 @@ use switchyard_protocol::{
Context, Decision, LlmClientError, LlmResponse, Request, Response, text_response,
};
-use crate::core::algorithm::{Algorithm, CallLlmRequest, RunObserver};
+use crate::core::algorithm::{Algorithm, CallLlmRequest};
use crate::{LibsyError, Result};
/// The result a fake client hands back for one offloaded call.
@@ -51,25 +51,14 @@ where
}
/// Run `algorithm` to completion, serving each offloaded call with `serve`.
-pub(crate) async fn drive(
+pub(crate) async fn test_drive(
algorithm: Arc<dyn Algorithm>,
ctx: Context,
request: Request,
serve: impl Serve,
-) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
- drive_observed(algorithm, ctx, request, None, serve).await
-}
-
-/// [`drive`] plus a [`RunObserver`], for tests that assert on run observations.
-pub(crate) async fn drive_observed(
- algorithm: Arc<dyn Algorithm>,
- ctx: Context,
- request: Request,
- observer: Option<RunObserver>,
- serve: impl Serve,
) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
let serve = Arc::new(serve);
- crate::drive(algorithm, ctx, request, observer, move |call| {
+ crate::drive(algorithm, ctx, request, move |call| {
fulfill(Arc::clone(&serve), call)
})
.await
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index d6593d23f..fdbcd5e0d 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -6,8 +6,8 @@
mod core;
pub use core::algorithm::{
- Algorithm, CallLlmRequest, Driver, LlmCallObservation, LlmTarget, LlmTargetSet, RoutedRequest,
- RunObservation, RunObserver, Step, StepStream, drive,
+ Algorithm, CallLlmRequest, Driver, LlmTarget, LlmTargetSet, RoutedRequest, Step, StepStream,
+ drive,
};
pub use core::classifier::{Classification, Classifier, Score};
pub use core::processor::{Event, Processor};
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 1e02bcc88..272a2e5fd 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -41,7 +41,7 @@ use opentelemetry::metrics::{Meter, ObservableGauge};
use opentelemetry::{KeyValue, global};
use tracing::Span;
-use crate::{Driver, Result};
+use crate::Result;
use switchyard_protocol::{Context, Decision, Request, Response};
const METRICS_SCOPE: &str = "switchyard";
@@ -146,12 +146,10 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {
}
/// Runs one algorithm task to completion, recording the run counter, duration
-/// histogram, routing overhead, span outcome, and failure log when it resolves.
+/// histogram, span outcome, and failure log when it resolves.
/// Executes inside the `libsy.run` span its caller instruments the task with.
-/// `driver` is the run's own, holding the duration of the call that served it.
pub(crate) async fn observe_run(
ctx: Context,
- driver: Driver,
run: impl Future<Output = Result<Response>>,
) -> Result<Response> {
let started = Instant::now();
@@ -159,12 +157,6 @@ pub(crate) async fn observe_run(
let duration = started.elapsed();
let algorithm = algorithm_label(&ctx);
record_run(algorithm, duration, &result, &Span::current());
- if result.is_ok()
- && let Some(overhead) =
- record_routing_overhead(algorithm, duration, driver.routed_call_duration())
- {
- driver.observe_routing_overhead(overhead);
- }
result
}
@@ -199,28 +191,6 @@ fn record_run(algorithm: &str, duration: Duration, result: &Result<Response>, sp
.record(duration.as_secs_f64() * 1000.0, &attributes);
}
-/// Records what routing cost on top of the call that served the run: classifier
-/// calls, target resolution, decision publishing. A run with no routed call has
-/// nothing to subtract, so it records nothing.
-fn record_routing_overhead(
- algorithm: &str,
- run: Duration,
- routed_call: Option<Duration>,
-) -> Option<Duration> {
- let routed_call = routed_call?;
- // Saturating: the two clocks start a moment apart, so a run that is all
- // routed call can come out fractionally negative.
- let overhead = run.saturating_sub(routed_call);
- meter()
- .f64_histogram("switchyard.routing_overhead_ms")
- .build()
- .record(
- overhead.as_secs_f64() * 1000.0,
- &[KeyValue::new("algorithm", algorithm.to_string())],
- );
- Some(overhead)
-}
-
/// Records a judge failure that made the classifier route without a verdict.
pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static str) {
meter()
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 4e4b28993..2c9414687 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -32,11 +32,11 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Extension, Json, Router};
use axum_server::tls_rustls::RustlsConfig;
-use libsy::{Algorithm, LibsyError, RunObservation, RunObserver};
+use libsy::{Algorithm, LibsyError};
use parking_lot::Mutex;
use serde::Deserialize;
use serde_json::{Value, json};
-use switchyard_llm_client::{ClientRouter, TranslatingLlmClient};
+use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};
use switchyard_protocol::{Context, Decision, LlmClientError, Metadata, Request, Usage};
use tokio::net::{TcpListener, TcpSocket};
use tokio::task;
@@ -689,16 +689,14 @@ async fn handle_llm_request(
Err(response) => return response,
};
let algorithm = Arc::clone(&route.algorithm);
- // Not one client for the route: a router that resolves each offloaded call to the
- // client configured for the target the algorithm selected.
- let target_clients = route.target_clients.clone();
+ let client_router = route.target_clients.clone();
let observer = stats_observer(
state.stats.clone(),
state.routing_log.clone().zip(routing_log_context.clone()),
);
let (trace, response) = match switchyard_llm_client::run(
algorithm,
- target_clients,
+ client_router,
Context::default(),
request,
Some(observer),
@@ -1318,7 +1316,7 @@ fn endpoint_listing(has_routing_log: bool) -> String {
#[cfg(test)]
mod tests {
- use libsy::LlmCallObservation;
+ use switchyard_llm_client::LlmCallObservation;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::{Notify, oneshot};
diff --git a/docs/internal/metrics_reference.md b/docs/internal/metrics_reference.md
index 9185b047b..6487d495f 100644
--- a/docs/internal/metrics_reference.md
+++ b/docs/internal/metrics_reference.md
@@ -57,7 +57,7 @@ Each histogram emits `_bucket`, `_sum`, and `_count` series. Use
| Metric | Type | Meaning |
|---|---|---|
-| `switchyard_routing_overhead_ms{algorithm}` | histogram | Algorithm run time minus the final routed model call. Includes classifier calls, target resolution, and decision publication; runs with no final routed call are not recorded. |
+| `switchyard_routing_overhead_ms{algorithm}` | histogram | Total run time minus the time spent in successful routed model calls, with overlapping hedged calls counted once. Includes classifier calls, failed routed attempts, target resolution, and decision publication; runs with no successful routed call are not recorded. Measured across the whole run, so it does not reconcile with `switchyard_run_duration_ms`, which times only the algorithm task. |
## Classifier fail-open counter
@@ -160,6 +160,6 @@ into label space.
|---|---|
| `model="<unknown>"` rows appear | A routed-call observation did not include a selected model. |
| All counters at 0 after warm-up | Server just started with no traffic, or the scraper is hitting the wrong port. |
-| `switchyard_routing_overhead_ms_count` stuck at `0` | No successful algorithm run has recorded a final routed model call. |
+| `switchyard_routing_overhead_ms_count` stuck at `0` | No successful algorithm run has recorded a successful routed model call. |
| `switchyard_classifier_fail_open_total` rising | The judge target is failing or returning a response the classifier cannot parse. Check `judge_model` and `reason`. |
| `switchyard_client_responses_total{outcome="retryable_error"}` rising | Either the upstream is genuinely flaky, or retries are exhausting; compare client responses with retryable upstream attempts. |