Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): count routing-time model calls that stop the run (#830)
Graham Kingc6e5958a663a
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 01e1c36fc..73a541f1c 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -185,7 +185,7 @@ async fn serve(};let target = call.models.first().ok_or(LibsyError::NoTargets)?;let request = clients.prepare_routing_request(call.request.clone(), target);- let response = call_one(+ match call_one(&clients,target,request,@@ -195,8 +195,11 @@ async fn serve(call.models.len(),true,)- .await?;- call.respond(Ok(response))+ .await+ {+ Ok(response) => call.respond(Ok(response)),+ Err(error) => call.fail(error),+ }}/// Try candidates in order until one succeeds or a failure stops fallback.diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 8150564f9..7622efe81 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -1718,6 +1718,23 @@ async fn classifier_stops_on_client_errors_and_records_verdict_fallback()}let snapshots = flushed_metrics(exporter, provider);+ let outcome = match client.outcome {+ JudgeOutcome::CallFailure | JudgeOutcome::StreamDecodeFailure => "error",+ JudgeOutcome::Reply(_) => "ok",+ };+ assert_eq!(+ u64_counter_value(+ &snapshots,+ "switchyard.llm_calls",+ &[+ ("algorithm", "llm_task_classifier"),+ ("selected_model", judge_model),+ ("outcome", outcome),+ ],+ ),+ Some(1),+ "logical call accounting for {judge_model}"+ );match expected_reason {Some(reason) => assert_eq!(u64_counter_value(diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 119809278..959621db3 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -109,19 +109,51 @@ pub struct CallModel {pub request: Request,/// Candidate models, tried in order until one answers. Never empty.pub models: Vec<ModelId>,- // How to send the response back to the algorithm- reply: oneshot::Sender<Result<Response>>,+ /// How to send the response back to the algorithm. `None` once the call is recorded.+ reply: Option<oneshot::Sender<Result<Response>>>,+ started: Instant,}impl CallModel {/// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to/// propagate a failed model call back to the algorithm. Consumes the promise: it/// can only be fulfilled once.- pub fn respond(self, result: Result<Response>) -> Result<()> {+ pub fn respond(mut self, result: Result<Response>) -> Result<()> {+ self.record(result.is_ok());self.reply+ .take()+ .ok_or(DriverError::ResponseDropped)?.send(result).map_err(|_| DriverError::ResponseDropped.into())}++ /// Record a failed call and return its error to stop [`drive`].+ /// Leaves the promise unfulfilled so the driver can cancel the algorithm.+ pub fn fail(mut self, error: LibsyError) -> Result<()> {+ self.reply = None;+ self.record(false);+ Err(error)+ }++ fn record(&self, is_ok: bool) {+ observability::record_llm_call(+ &self.algorithm,+ self.models+ .first()+ .map(ModelId::as_str)+ .unwrap_or("NoTargets"),+ self.started.elapsed(),+ is_ok,+ );+ }+}++impl Drop for CallModel {+ fn drop(&mut self) {+ if self.reply.is_some() {+ self.record(false);+ }+ }}/// The terminal result of routing.@@ -239,8 +271,9 @@ impl Driver {/// Errors if the stream is closed or the call failed./// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as/// the algorithm observes it (host queueing/serving included; a streamed- /// response resolves when its stream handle arrives); latency, outcome, and- /// token usage are recorded when it resolves. The provider call itself is the+ /// response resolves when its stream handle arrives). The host records call metrics+ /// through [`CallModel::respond`] or [`CallModel::fail`]; outcome and token usage+ /// are recorded on the span when the promise resolves. The provider call itself is the/// host's, and is instrumented by whoever makes it.#[tracing::instrument(target = "libsy",@@ -258,7 +291,7 @@ impl Driver {))]pub async fn call_model(&self, mut request: Request, models: Vec<ModelId>) -> Result<Response> {- let Some(selected_model_id) = models.first().cloned() else {+ let Some(selected_model_id) = models.first() else {return Err(LibsyError::NoTargets);};request.llm_request.model = Some(selected_model_id.to_string());@@ -268,7 +301,8 @@ impl Driver {algorithm: self.algorithm.clone(),request,models,- reply,+ reply: Some(reply),+ started,};let result = async {self.step_tx@@ -280,14 +314,7 @@ impl Driver {.map_err(|_| LibsyError::from(DriverError::ResponseDropped))?}.await;- let elapsed = started.elapsed();- observability::record_llm_call(- &self.algorithm,- selected_model_id.as_str(),- elapsed,- &result,- &tracing::Span::current(),- );+ observability::record_llm_call_span(&result, &tracing::Span::current());result}@@ -363,9 +390,10 @@ pub enum Step {/// Returns the final [`RoutingOutcome`]./// `serve` owns the call: it performs it however the host likes and must fulfill the promise/// with [`CallModel::respond`]. A failed *model* call belongs in `respond` — the-/// algorithm may route around it. Returning `Err` from `serve` aborts the whole run, so-/// reserve it for infrastructure failures. Calls are served concurrently, so an algorithm-/// that offloads several at once (hedging, fan-out) gets real parallelism.+/// algorithm may route around it. To stop routing on a model-call failure, return+/// [`CallModel::fail`] instead. Returning `Err` from `serve` aborts the whole run.+/// Calls are served concurrently, so an algorithm that offloads several at once (hedging, fan-out)+/// gets real parallelism.////// libsy performs no I/O; this is only the mechanics of consuming its own step stream, kept/// here so every host does not reimplement the same loop. `switchyard-llm-client`'s `run`diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 69c6d9295..4779540da 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -232,18 +232,14 @@ pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static st);}-/// Records the resolution of one offloaded model call: the call counter and-/// latency histogram and the outcome/token fields on `span`, without error details.+/// Records the call counter and latency histogram for a completed offloaded call.pub(crate) fn record_llm_call(algorithm: &str,selected_model: &str,duration: Duration,- result: &Result<Response>,- span: &Span,+ is_ok: bool,) {- let outcome = outcome_value(result);- span.record("outcome", outcome);-+ let outcome = if is_ok { "ok" } else { "error" };let meter = meter();let call_attributes = [KeyValue::new("algorithm", algorithm.to_string()),@@ -258,7 +254,11 @@ pub(crate) fn record_llm_call(.f64_histogram("switchyard.llm_call_duration_ms").build().record(duration.as_secs_f64() * 1000.0, &call_attributes);+}+/// Records the outcome and token fields on the algorithm's call span.+pub(crate) fn record_llm_call_span(result: &Result<Response>, span: &Span) {+ span.record("outcome", outcome_value(result));if let Ok(response) = result {// Token usage exists only once a response is buffered; a streamed// response resolves before its usage is known, so none is recorded.