Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): cancel abandoned offloaded calls (#948)
nachiketb-nvidia50f635d74c72
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.mdindex 19a423c16..c2c948150 100644--- a/crates/libsy-llm-client/README.md+++ b/crates/libsy-llm-client/README.md@@ -154,6 +154,12 @@ async fn stream(runs. The terminal outcome either already contains the answer or supplies the selected model andordered fallbacks for the client to try. `ClientRouter::single` is the single-provider case:+Routing-time calls run concurrently. If an algorithm drops one of its waiting+call futures, the corresponding HTTP work is dropped while other calls continue.+Ending or dropping the run drops all outstanding call handlers. A returned+response stream stays alive until its consumer finishes or drops it. These are+local cancellation guarantees; a provider may continue generation after disconnect.+```rustuse std::sync::Arc;use switchyard_libsy::{Algorithm, RuntimeModels};diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex dcb4a6f7d..ab8a3e9cb 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -163,21 +163,25 @@ async fn serve_decision(observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,) -> Result<()> {let client = clients.route_decision(&call.model);- let started = Instant::now();- let result = async { client?.call(call.request.clone()).await }.await;- tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });- if let Some(observations) = observations {- observations- .lock()- .push(RunObservation::DecisionCall(ModelCallObservation {- selected_model: call.model.clone(),- is_success: result.is_ok(),- duration: started.elapsed(),- usage: result.as_ref().ok().map(|response| response.usage.clone()),- }));- }- let result = result.map_err(|error| LibsyError::client_call(call.model.clone(), error));- call.respond(result)+ let model = call.model.clone();+ let request = call.request.clone();+ call.respond(async {+ let started = Instant::now();+ let result = async { client?.call(request).await }.await;+ tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });+ if let Some(observations) = observations {+ observations+ .lock()+ .push(RunObservation::DecisionCall(ModelCallObservation {+ selected_model: model.clone(),+ is_success: result.is_ok(),+ duration: started.elapsed(),+ usage: result.as_ref().ok().map(|response| response.usage.clone()),+ }));+ }+ result.map_err(|error| LibsyError::client_call(model, error))+ })+ .await}/// Emits completed routing calls after the outcome reveals whether one response became the answer.@@ -223,24 +227,14 @@ async fn serve(.push(RunObservation::LlmCall(observation));}};- let target = call.models.first().ok_or(LibsyError::NoTargets)?;- let request = clients.prepare_routing_request(call.request.clone(), target);- match call_one(- &clients,- target,- request,- &call.algorithm,- &observe,- 0,- call.models.len(),- true,- )+ let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;+ let request = clients.prepare_routing_request(call.request.clone(), &target);+ let algorithm = call.algorithm.clone();+ let candidates = call.models.len();+ call.respond(call_one(+ &clients, &target, request, &algorithm, &observe, 0, candidates, true,+ )).await- {- Ok(response) => call.respond(Ok(response)),- Err(error) if call.recover_errors => call.respond(Err(error)),- Err(error) => call.fail(error),- }}/// Try candidates in order until one succeeds or a failure stops fallback.diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 30366dbb2..37a71f3db 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -1660,17 +1660,23 @@ async fn decision_calls_record_each_terminal_path_once() -> switchyard_libsy::Redrop(call);} else {match mode {- "reply" => call.respond(Ok(DecisionResponse {- id: Some("decision-response".into()),- model: Some("provider-model".into()),- answers: Default::default(),- usage: Usage {- input_tokens: Some(42),- output_tokens: Some(5),- ..Usage::default()- },- }))?,- "error" => call.respond(Err(test_error(LEAKED_CONTENT)))?,+ "reply" => {+ call.respond(std::future::ready(Ok(DecisionResponse {+ id: Some("decision-response".into()),+ model: Some("provider-model".into()),+ answers: Default::default(),+ usage: Usage {+ input_tokens: Some(42),+ output_tokens: Some(5),+ ..Usage::default()+ },+ })))+ .await?+ }+ "error" => {+ call.respond(std::future::ready(Err(test_error(LEAKED_CONTENT))))+ .await?+ }"fail" => assert!(call.fail(test_error(LEAKED_CONTENT)).is_err()),"drop" => drop(call),_ => unreachable!(),@@ -1757,8 +1763,12 @@ async fn failed_call_records_metrics_without_error_details() -> switchyard_libsylet mut saw_error_step = false;while let Some(step) = stream.next().await {match step {- Ok(Step::CallModel(call)) => {- call.respond(Err(test_error("synthetic upstream failure")))?;+ Ok(Step::CallModel(mut call)) => {+ call.recover_errors = true;+ call.respond(std::future::ready(Err(test_error(+ "synthetic upstream failure",+ ))))+ .await?;}Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),Ok(Step::Done(_)) => {@@ -2131,11 +2141,12 @@ async fn in_flight_gauge_reads_a_run_parked_on_an_unanswered_routing_call()"the run counter must not report a run that has not resolved");- call.respond(Ok(Response {+ call.respond(std::future::ready(Ok(Response {llm_response: LlmResponse::Agg(text_response(Some(MODEL.to_string()), "answer")),metadata: None,upstream_headers: http::HeaderMap::new(),- }))?;+ })))+ .await?;while stream.next().await.is_some() {}let snapshots = flushed_metrics(exporter, provider);diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex a7df749ee..0be56eb61 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -41,6 +41,28 @@ drives the stream and performs the terminal answer call, retries, and fallback oThe provider-neutral [`Request`], [`Response`], [`Usage`], and [`LlmResponse`]contracts come from `switchyard-protocol`.+## Concurrent calls and cancellation++An algorithm can poll several `Driver::call_model` or `Driver::call_decision` futures+with `tokio::join!`, `tokio::select!`, or `FuturesUnordered`. Each call has its own+reply channel. Dropping a waiting future cancels only that call; the algorithm can+continue making other calls.++Hosts pass the work itself to `call.respond(work).await`. Awaiting the provider+before calling `respond` prevents libsy from cancelling that work. For a response+that is already available, use `call.respond(std::future::ready(result)).await`.+Model-call errors stop the host run unless `call.recover_errors` is enabled.+Decision-call errors are returned to the algorithm.++`drive` polls host handlers concurrently and drops outstanding handlers when the+algorithm finishes or the run is dropped. Custom `run_stream` consumers must also+poll handlers concurrently and drop them when the run ends. Separately spawned+tasks need explicit abort and cleanup; dropping a task handle does not stop its work.++Cancellation drops the supplied work future. Once a response stream is delivered,+its consumer controls its lifetime. Whether the provider stops generation depends+on the transport and provider.+[`Request`]: switchyard_protocol::Request[`Response`]: switchyard_protocol::Response[`Usage`]: switchyard_protocol::Usagediff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rsindex 8bf901595..b59e549e8 100644--- a/crates/libsy/src/algorithms/advisor_gate/tests.rs+++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs@@ -356,7 +356,7 @@ async fn calls_preserve_candidates_and_attribute_the_serving_executor() {};let mut response = reply(text);response.set_served_model(&served);- call.respond(Ok(response))+ call.respond(std::future::ready(Ok(response))).await}}).awaitdiff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rsindex 7e3d15b83..cced4d9fb 100644--- a/crates/libsy/src/algorithms/escalation.rs+++ b/crates/libsy/src/algorithms/escalation.rs@@ -634,7 +634,7 @@ mod tests {.serve(target.clone(), request).await.map_err(|source| LibsyError::client_call(target, source));- call.respond(response)+ call.respond(std::future::ready(response)).await}},)diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex e45d73ff7..a95683790 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -1408,10 +1408,12 @@ mod tests {}}if name == "provider error" {- return call.respond(Err(LibsyError::client_call(- "judge",- LlmClientError::General("private provider body".into()),- )));+ return call+ .respond(std::future::ready(Err(LibsyError::client_call(+ "judge",+ LlmClientError::General("private provider body".into()),+ ))))+ .await;}if name == "dropped reply" {drop(call);@@ -1441,12 +1443,13 @@ mod tests {},)])};- call.respond(Ok(DecisionResponse {+ call.respond(std::future::ready(Ok(DecisionResponse {id: None,model: Some("provider-judge".into()),answers,usage: Default::default(),- }))+ })))+ .await}};let models = Arc::new(RuntimeModels::new(runtime_models()));diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 70ff7ef1e..2ca10cc82 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -588,7 +588,7 @@ mod tests {let serve = async {if let Some(Ok(Step::CallModel(call))) = steps.next().await {- let _ = call.respond(reply);+ let _ = call.respond(std::future::ready(reply)).await;}};let (classification, ()) =diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 133f8fbe4..6e45c1267 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -96,8 +96,8 @@ enum Scope {/// An offloaded model call, surfaced inside [`Step::CallModel`].///-/// The host reads the public fields, performs (or delegates) the model call, and fulfills it-/// with [`respond`](Self::respond) — unblocking the algorithm's [`Driver::call_model`] on the+/// The host reads the public fields and passes its model-call future to+/// [`respond`](Self::respond), unblocking the algorithm's [`Driver::call_model`] on the/// other side. `switchyard-llm-client`'s `run` is the ready-made consumer that does this for/// you.///@@ -121,10 +121,26 @@ pub struct CallModel {}impl CallModel {- /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to- /// propagate a failed model call back to the algorithm. Consumes the promise: it- /// can only be fulfilled once.- pub fn respond(mut self, result: Result<Response>) -> Result<()> {+ /// Run host work until it completes or the algorithm stops waiting for this call.+ /// Dropping the waiting [`Driver::call_model`] future drops `work` and returns `Ok(())`.+ /// Errors stop the host's run unless [`Self::recover_errors`] is enabled.+ /// Use `std::future::ready(result)` for an already available response.+ /// A delivered response stream owns its subsequent lifetime.+ pub async fn respond(mut self, work: impl Future<Output = Result<Response>>) -> Result<()> {+ let reply = self.reply.as_mut().ok_or(DriverError::ResponseDropped)?;+ let result = tokio::select! {+ biased;+ _ = reply.closed() => return Ok(()),+ result = work => result,+ };+ if self.reply.as_ref().is_some_and(oneshot::Sender::is_closed) {+ return Ok(());+ }+ if !self.recover_errors+ && let Err(error) = result+ {+ return self.fail(error);+ }self.record(result.is_ok());if let Ok(response) = &result {observability::record_llm_response(response, &self.span);@@ -133,7 +149,8 @@ impl CallModel {.take().ok_or(DriverError::ResponseDropped)?.send(result)- .map_err(|_| DriverError::ResponseDropped.into())+ .ok();+ Ok(())}/// Record a failed call and return its error to stop [`drive`].@@ -183,8 +200,19 @@ pub struct CallDecision {}impl CallDecision {- /// Return a response or provider error to the algorithm so it can continue or fall back.- pub fn respond(mut self, result: Result<DecisionResponse>) -> Result<()> {+ /// Run host work and return its response or error to the algorithm.+ /// Dropping the waiting [`Driver::call_decision`] future drops `work` and returns `Ok(())`.+ /// Use `std::future::ready(result)` for an already available response.+ pub async fn respond(+ mut self,+ work: impl Future<Output = Result<DecisionResponse>>,+ ) -> Result<()> {+ let reply = self.reply.as_mut().ok_or(DriverError::ResponseDropped)?;+ let result = tokio::select! {+ biased;+ _ = reply.closed() => return Ok(()),+ result = work => result,+ };self.record(result.is_ok());if let Ok(response) = &result {observability::record_decision_response(response, &self.span);@@ -193,7 +221,8 @@ impl CallDecision {.take().ok_or(DriverError::ResponseDropped)?.send(result)- .map_err(|_| DriverError::ResponseDropped.into())+ .ok();+ Ok(())}/// Returning this error from the host handler aborts [`drive`].@@ -862,11 +891,11 @@ mod tests {calls.remove("second").ok_or_else(|| test_error("missing second call"))?- .respond(Ok(reply("second response")))?;+ .respond(std::future::ready(Ok(reply("second response")))).await?;calls.remove("first").ok_or_else(|| test_error("missing first call"))?- .respond(Ok(reply("first response")))?;+ .respond(std::future::ready(Ok(reply("first response")))).await?;let first_response = first.await@@ -998,7 +1027,7 @@ mod tests {async move {barrier.wait().await;let call = match call {- Call::Model(call) => return call.respond(Ok(reply("llm reply"))),+ Call::Model(call) => return call.respond(std::future::ready(Ok(reply("llm reply")))).await,Call::Decision(call) => *call,};assert_eq!(call.algorithm, "mixed");@@ -1007,10 +1036,10 @@ mod tests {assert_eq!(call.request.context, decision_request().context);match mode {"mock" => serve_decision(call).await,- "reply" => call.respond(Ok(decision_response())),- "error" => call.respond(Err(LibsyError::AlgorithmError {+ "reply" => call.respond(std::future::ready(Ok(decision_response()))).await,+ "error" => call.respond(std::future::ready(Err(LibsyError::AlgorithmError {message: "provider failed".into(),- })),+ }))).await,"drop" => {drop(call);Ok(())@@ -1036,10 +1065,7 @@ mod tests {return Err(test_error("expected a decision call"));};drop(pending);- assert!(matches!(- call.respond(Ok(decision_response())),- Err(LibsyError::Driver(DriverError::ResponseDropped))- ));+ call.respond(async { panic!("cancelled work must not be polled") }).await?;drop(steps);assert!(matches!(driver.call_decision(decision_request(), "decision".into()).await,@@ -1153,14 +1179,15 @@ mod tests {saw_call = true;assert_eq!(call.models, vec![ModelId::from("offload/model")]);// Fulfilling the promise is the "real" model call the caller makes.- call.respond(Ok(Response {+ call.respond(std::future::ready(Ok(Response {llm_response: LlmResponse::Agg(text_response(None,"fulfilled".to_string(),)),metadata: None,upstream_headers: http::HeaderMap::new(),- }))?;+ })))+ .await?;}Step::Done(outcome) => {let metadata = outcome@@ -1263,8 +1290,12 @@ mod tests {while let Some(step) = stream.next().await {match step {Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),- Ok(Step::CallModel(call)) => {- call.respond(Err(test_error("upstream model call failed")))?;+ Ok(Step::CallModel(mut call)) => {+ call.recover_errors = true;+ call.respond(std::future::ready(Err(test_error(+ "upstream model call failed",+ ))))+ .await?;}Ok(Step::Done(..)) => {return Err(test_error(diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rsindex b62667cb3..d256a592e 100644--- a/crates/libsy/src/core/testing.rs+++ b/crates/libsy/src/core/testing.rs@@ -101,23 +101,27 @@ pub(crate) async fn serve_decision(call: CallDecision) -> Result<()> {answers: Default::default(),usage: Default::default(),};- call.respond(Ok(response))+ call.respond(std::future::ready(Ok(response))).await}/// Serve one call and fulfill its promise, mapping failures the way a host does so/// error-shape assertions match production.async fn fulfill(serve: Arc<impl Serve>, call: Call) -> Result<()> {- let call = match call {+ let mut call = match call {Call::Model(call) => *call,Call::Decision(call) => return serve_decision(*call).await,};let request = call.request.clone();let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;- let result = serve- .serve(target.clone(), request)- .await- .map_err(|source| LibsyError::client_call(target, source));- call.respond(result)+ // Test clients return failures to routing policies, including policies under test.+ call.recover_errors = true;+ call.respond(async {+ serve+ .serve(target.clone(), request)+ .await+ .map_err(|source| LibsyError::client_call(target, source))+ })+ .await}/// Answers with the selected model name as the completion — what most routing tests need,diff --git a/crates/libsy/tests/cancellation.rs b/crates/libsy/tests/cancellation.rsnew file mode 100644index 000000000..242404ad3--- /dev/null+++ b/crates/libsy/tests/cancellation.rs@@ -0,0 +1,132 @@+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+// SPDX-License-Identifier: Apache-2.0++use std::sync::Arc;+use std::time::Duration;++use async_trait::async_trait;+use switchyard_libsy::{Algorithm, Call, Driver, Result, RoutingOutcome, RuntimeModels};+use switchyard_protocol::{+ DecisionRequest, LlmResponse, Request, Response, text_request, text_response,+};+use tokio::sync::Notify;++struct Abandon {+ decision: bool,+ started: Arc<Notify>,+ dropped: Arc<Notify>,+}++#[async_trait]+impl Algorithm for Abandon {+ fn name(&self) -> &str {+ "abandon"+ }++ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome> {+ {+ let call = async {+ if self.decision {+ driver+ .call_decision(+ DecisionRequest {+ model: None,+ context: serde_json::json!({}),+ questions: Default::default(),+ },+ "abandoned".into(),+ )+ .await+ .map(|_| ())+ } else {+ driver+ .call_model(request.clone(), vec!["abandoned".into()])+ .await+ .map(|_| ())+ }+ };+ tokio::pin!(call);+ tokio::select! {+ _ = self.started.notified() => {},+ _ = &mut call => panic!("abandoned work finished"),+ }+ }+ // The next call cannot begin until the host drops the abandoned work.+ self.dropped.notified().await;+ let response = driver+ .call_model(request.clone(), vec!["continuation".into()])+ .await?;+ Ok(RoutingOutcome::answered(+ "continuation".into(),+ request,+ response,+ ))+ }+}++struct DropNotify(Arc<Notify>);+impl Drop for DropNotify {+ fn drop(&mut self) {+ self.0.notify_one();+ }+}++async fn pending<T>(started: Arc<Notify>, dropped: Arc<Notify>) -> Result<T> {+ let _guard = DropNotify(dropped);+ started.notify_one();+ std::future::pending().await+}++fn request() -> Request {+ Request {+ llm_request: text_request(None, "hello"),+ raw_request: None,+ metadata: None,+ }+}++fn response() -> Response {+ Response {+ llm_response: LlmResponse::Agg(text_response(None, "continued")),+ metadata: None,+ upstream_headers: Default::default(),+ }+}++#[tokio::test]+async fn abandoning_model_or_decision_work_cancels_only_that_call() {+ for decision in [false, true] {+ let started = Arc::new(Notify::new());+ let dropped = Arc::new(Notify::new());+ let algorithm = Arc::new(Abandon {+ decision,+ started: started.clone(),+ dropped: dropped.clone(),+ });+ let outcome = tokio::time::timeout(+ Duration::from_secs(1),+ switchyard_libsy::drive(+ algorithm,+ request(),+ Arc::new(RuntimeModels::default()),+ move |call| {+ let started = started.clone();+ let dropped = dropped.clone();+ async move {+ match call {+ Call::Model(call) if call.models[0] == "continuation" => {+ call.respond(std::future::ready(Ok(response()))).await+ }+ Call::Model(call) => call.respond(pending(started, dropped)).await,+ Call::Decision(call) => call.respond(pending(started, dropped)).await,+ }+ }+ },+ ),+ )+ .await+ .expect("cancelled call blocked continuation")+ .expect("cancelled call failed the run");+ assert_eq!(outcome.selected_model_id().unwrap(), "continuation");+ }+}diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rsindex bd66d230d..f1b338cf9 100644--- a/crates/prefill-router/tests/unit/algorithm.rs+++ b/crates/prefill-router/tests/unit/algorithm.rs@@ -72,7 +72,7 @@ async fn serve_decision(call: libsy::CallDecision) -> libsy::Result<()> {answers: Default::default(),usage: Default::default(),};- call.respond(Ok(response))+ call.respond(std::future::ready(Ok(response))).await}async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<String> {@@ -85,13 +85,14 @@ async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<Call::Model(call) => *call,Call::Decision(call) => return serve_decision(*call).await,};- call.respond(Ok(switchyard_protocol::Response {+ call.respond(std::future::ready(Ok(switchyard_protocol::Response {llm_response: switchyard_protocol::LlmResponse::Agg(switchyard_protocol::text_response(None, "unused"),),metadata: None,upstream_headers: Default::default(),- }))+ })))+ .await},).await?;diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 598c830f2..aa98c91db 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -542,8 +542,22 @@ impl PyModelCall {self.models.clone()}- /// Fulfill this call with a normalized aggregate or streamed response.- fn respond(&mut self, py: Python<'_>, response: PyRef<'_, PyLlmResponse>) -> PyResult<()> {+ /// Await a normalized response, cancelling Python work if the algorithm stops waiting.+ fn respond<'py>(+ slf: PyRef<'py, Self>,+ py: Python<'py>,+ work: &Bound<'py, PyAny>,+ ) -> PyResult<Bound<'py, PyAny>> {+ py.import("switchyard_rust._libsy_async")?+ .getattr("respond")?+ .call1((slf, work))+ }++ fn _respond<'py>(+ &mut self,+ py: Python<'py>,+ work: Bound<'py, PyAny>,+ ) -> PyResult<Bound<'py, PyAny>> {let model = self.inner.as_ref()@@ -554,36 +568,53 @@ impl PyModelCall {.as_ref().map(ModelId::new).ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;- let llm_response = response.to_core(py, model)?;+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;+ let work = pyo3_async_runtimes::into_future_with_locals(&locals, work)?;let call = self.take()?;let metadata = call.request.metadata.clone();- call.respond(Ok(Response {- llm_response,- metadata,- upstream_headers: HeaderMap::new(),- }))- .map_err(py_libsy_error)+ pyo3_async_runtimes::tokio::future_into_py(py, async move {+ call.respond(async move {+ let result = work.await;+ Python::attach(|py| {+ let response = result.map_err(|error| {+ RustLibsyError::client_call(+ model.clone(),+ python_client_error(py, error, &model),+ )+ })?;+ let response = response+ .bind(py)+ .extract::<PyRef<'_, PyLlmResponse>>()+ .map_err(|error| ffi_error(error.into()))+ .map_err(|source| RustLibsyError::client_call(model.clone(), source))?;+ let llm_response = response+ .to_core(py, model.clone())+ .map_err(ffi_error)+ .map_err(|source| RustLibsyError::client_call(model, source))?;+ Ok(Response {+ llm_response,+ metadata,+ upstream_headers: HeaderMap::new(),+ })+ })+ })+ .await+ .map_err(py_libsy_error)+ })}- /// Fulfill this call with a Python client failure.- fn fail(&mut self, error: &Bound<'_, PyAny>) -> PyResult<()> {+ /// Return a Python client failure through the same async reply path.+ fn fail<'py>(+ slf: PyRef<'py, Self>,+ py: Python<'py>,+ error: &Bound<'py, PyAny>,+ ) -> PyResult<Bound<'py, PyAny>> {if !error.is_instance_of::<PyBaseException>() {return Err(PyTypeError::new_err("error must derive from BaseException"));}- let target = self- .inner- .as_ref()- .ok_or_else(|| py_libsy_error("model call has already been completed"))?- .request- .llm_request- .model- .as_ref()- .map(ModelId::new)- .ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;- let call = self.take()?;- let source = python_client_error(error.py(), PyErr::from_value(error.clone()), &target);- call.respond(Err(RustLibsyError::client_call(target, source)))- .map_err(py_libsy_error)+ py.import("switchyard_rust._libsy_async")?+ .getattr("fail")?+ .call1((slf, error))}}@@ -632,22 +663,58 @@ impl PyDecisionCall {&self.model}- /// Return a provider-neutral decision response to the waiting algorithm.- fn respond(&mut self, response: &Bound<'_, PyAny>) -> PyResult<()> {- let response: DecisionResponse = from_python(response)?;- self.take()?.respond(Ok(response)).map_err(py_libsy_error)+ /// Await a decision response, cancelling Python work if the algorithm stops waiting.+ fn respond<'py>(+ slf: PyRef<'py, Self>,+ py: Python<'py>,+ work: &Bound<'py, PyAny>,+ ) -> PyResult<Bound<'py, PyAny>> {+ py.import("switchyard_rust._libsy_async")?+ .getattr("respond")?+ .call1((slf, work))+ }++ fn _respond<'py>(+ &mut self,+ py: Python<'py>,+ work: Bound<'py, PyAny>,+ ) -> PyResult<Bound<'py, PyAny>> {+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;+ let work = pyo3_async_runtimes::into_future_with_locals(&locals, work)?;+ let call = self.take()?;+ let model = call.model.clone();+ pyo3_async_runtimes::tokio::future_into_py(py, async move {+ call.respond(async move {+ let result = work.await;+ Python::attach(|py| {+ let response = result.map_err(|error| {+ RustLibsyError::client_call(+ model.clone(),+ python_client_error(py, error, &model),+ )+ })?;+ from_python::<DecisionResponse>(response.bind(py))+ .map_err(ffi_error)+ .map_err(|source| RustLibsyError::client_call(model, source))+ })+ })+ .await+ .map_err(py_libsy_error)+ })}/// Return a Python client failure so the algorithm can apply its fallback policy.- fn fail(&mut self, error: &Bound<'_, PyAny>) -> PyResult<()> {+ fn fail<'py>(+ slf: PyRef<'py, Self>,+ py: Python<'py>,+ error: &Bound<'py, PyAny>,+ ) -> PyResult<Bound<'py, PyAny>> {if !error.is_instance_of::<PyBaseException>() {return Err(PyTypeError::new_err("error must derive from BaseException"));}- let source = python_client_error(error.py(), PyErr::from_value(error.clone()), &self.model);- let model = self.model.clone();- self.take()?- .respond(Err(RustLibsyError::client_call(model, source)))- .map_err(py_libsy_error)+ py.import("switchyard_rust._libsy_async")?+ .getattr("fail")?+ .call1((slf, error))}}diff --git a/docs/getting_started.md b/docs/getting_started.mdindex 5d524765e..41c1a0d23 100644--- a/docs/getting_started.md+++ b/docs/getting_started.md@@ -236,6 +236,42 @@ response when routing already produced the answer. Otherwise the host makes thecall from that outcome. Serving these calls yourself is what lets libsy embed in a host thatalready owns its HTTP stack, retries, and credentials.+Pass each call's work to `respond` before awaiting it. In Rust:++```rust+use switchyard_libsy::{CallModel, LibsyError, Result};+use switchyard_protocol::RoutedLlmClient;++async fn serve(call: CallModel, client: &dyn RoutedLlmClient) -> Result<()> {+ let request = call.request.clone();+ let model = call.models[0].clone();+ call.respond(async move {+ client.call(request).await.map_err(|error| LibsyError::client_call(model, error))+ }).await+}+```++In Python, the client coroutine returns `LlmResponse.Agg` or `LlmResponse.Stream`:++```python+await call.respond(client.call(call.request, call.models[0]))+```++`respond` now accepts a future or awaitable instead of a completed response. Move+the provider await into this method. Python `call.fail(error)` is also awaitable.+Decision calls use the same pattern with their existing decision response type.++When an algorithm stops waiting for a call, `respond` cancels that work and+returns normally. Python waits for task cleanup before returning. For concurrent+algorithms, schedule each Python handler with `asyncio.create_task` and cancel and+await remaining handlers when the run ends; see the complete+[Python example](../examples/libsy.py). Awaiting handlers one at a time inside the+step loop serializes their execution. Rust hosts can use `libsy::drive` to manage+concurrent handlers and their lifetime.++After a response stream is delivered, its consumer controls its lifetime.+Cancellation cannot guarantee that an upstream provider stops generating or billing.+Successful runs also include `OutcomeMetadata`: a unique `outcome_id`, the algorithmname, and optional JSON evidence. In Python, read `outcome.metadata.outcome_id`,`outcome.metadata.algorithm`, and `outcome.metadata.evidence` after checking thatdiff --git a/docs/reference/opentelemetry.md b/docs/reference/opentelemetry.mdindex 2d798b41d..34c8e975b 100644--- a/docs/reference/opentelemetry.md+++ b/docs/reference/opentelemetry.md@@ -173,7 +173,7 @@ Advisor Gate instruments use the prefix `switchyard.advisor_gate.`:### Timing and streaming- `libsy.run` may finish before the answer call. Nested algorithms have separate run spans.-- Decision calls record `error` when failed or dropped without a reply, including cancellation. A returned response records its outcome even if the waiting algorithm has already gone away. These metrics work with any host and do not require a client observer.+- Decision calls record `error` when failed or dropped without a reply, including cancellation. Cancelled calls do not poll further host work. A response completed during a cancellation race may still record its result. These metrics work with any host and do not require a client observer.- `libsy.llm_call` and routing-time call metrics end when the host fulfills the offloaded call. They include host queueing. The LLM client driver buffers routing streams before fulfilling the call. The span's `input_tokens`, `output_tokens`, `total_tokens`, and `reasoning_tokens` fields are buffered-response only.- For terminal answer candidates, `switchyard.llm_calls` and `switchyard.llm_call_duration_ms` record when the response stream ends or is dropped. Duration includes that candidate's retries and stream consumption. Stream errors and unfinished drops record `outcome=error`; a terminal message permits a successful drop.- `libsy.client_call` remains open while its stream is consumed. IDs, usage, and finish reasons update from normalized events. An unfinished stream dropped by its consumer records `cancelled`; a stream error records `error`.diff --git a/examples/libsy.py b/examples/libsy.pyindex 42da987df..719341ad2 100644--- a/examples/libsy.py+++ b/examples/libsy.py@@ -7,7 +7,7 @@import asynciofrom collections.abc import AsyncIterator, Mapping-from switchyard.libsy import LlmResponse, Step, algorithms+from switchyard.libsy import LlmResponse, RoutingOutcome, Step, algorithmsclass EchoClient:@@ -62,22 +62,33 @@ async def main() -> None:seed=42,)- async for step in algorithm.run_stream(request, {"any": ["fast", "quality"]}):- match step:- case Step.CallModel(call):- call.respond(await client.call(call.request, call.models[0]))- case Step.Done(outcome):- print("Decision:", outcome.selected_model_ids[0])- response = outcome.response or await client.call(- outcome.request,- outcome.selected_model_ids[0],- )- match response:- case LlmResponse.Agg(aggregate_response):- print("Response:", aggregate_response)- case LlmResponse.Stream(response_stream):- async for event in response_stream:- print("Response event:", event)+ handlers: list[asyncio.Task[None]] = []+ outcome: RoutingOutcome | None = None+ try:+ async for step in algorithm.run_stream(request, {"any": ["fast", "quality"]}):+ match step:+ case Step.CallModel(call):+ handlers.append(+ asyncio.create_task(call.respond(client.call(call.request, call.models[0])))+ )+ case Step.Done(done):+ outcome = done+ break+ finally:+ for handler in handlers:+ handler.cancel()+ await asyncio.gather(*handlers, return_exceptions=True)++ if outcome is None:+ raise RuntimeError("Algorithm ended without a routing outcome")+ print("Decision:", outcome.selected_model_ids[0])+ response = outcome.response or await client.call(outcome.request, outcome.selected_model_ids[0])+ match response:+ case LlmResponse.Agg(aggregate_response):+ print("Response:", aggregate_response)+ case LlmResponse.Stream(response_stream):+ async for event in response_stream:+ print("Response event:", event)if __name__ == "__main__":diff --git a/switchyard_rust/_libsy_async.py b/switchyard_rust/_libsy_async.pynew file mode 100644index 000000000..07ea10194--- /dev/null+++ b/switchyard_rust/_libsy_async.py@@ -0,0 +1,31 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Keep Python call tasks alive only while their libsy reply is needed."""++from __future__ import annotations++import asyncio+from collections.abc import Awaitable+from typing import Any, Protocol+++class _Call(Protocol):+ def _respond(self, task: asyncio.Future[Any]) -> Awaitable[None]: ...+++async def respond(call: _Call, work: Awaitable[Any]) -> None:+ task = asyncio.ensure_future(work)+ try:+ await call._respond(task)+ finally:+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+++async def fail(call: _Call, error: BaseException) -> None:+ async def raise_error() -> None:+ raise error++ await respond(call, raise_error())diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex af56c7df8..15ff7fc49 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -36,7 +36,7 @@ _EXPORTS = frozenset()if TYPE_CHECKING:- from collections.abc import AsyncIterator, Sequence+ from collections.abc import AsyncIterator, Awaitable, Sequencefrom typing import ClassVar, Literal, finalclass LibsyError(RuntimeError): ...@@ -130,9 +130,9 @@ if TYPE_CHECKING:@propertydef models(self) -> list[str]: ...- def respond(self, response: LlmResponse.Agg | LlmResponse.Stream) -> None: ...+ async def respond(self, work: Awaitable[LlmResponse.Agg | LlmResponse.Stream]) -> None: ...- def fail(self, error: BaseException) -> None: ...+ async def fail(self, error: BaseException) -> None: ...@finalclass DecisionCall:@@ -145,9 +145,9 @@ if TYPE_CHECKING:@propertydef model(self) -> str: ...- def respond(self, response: Mapping[str, object]) -> None: ...+ async def respond(self, work: Awaitable[Mapping[str, object]]) -> None: ...- def fail(self, error: BaseException) -> None: ...+ async def fail(self, error: BaseException) -> None: ...@finalclass OutcomeMetadata:diff --git a/tests/test_libsy_cancellation.py b/tests/test_libsy_cancellation.pynew file mode 100644index 000000000..150f47b56--- /dev/null+++ b/tests/test_libsy_cancellation.py@@ -0,0 +1,118 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Cancellation crosses the Python/Rust boundary and awaits Python cleanup."""++import asyncio+import gc++import pytest++from switchyard.libsy import (+ ContextWindowExceededError,+ LibsyError,+ LlmResponse,+ Step,+ TaskClassifierConfig,+ algorithms,+)++[email protected]("decision", [False, True])[email protected]("cancel_handler", [False, True])+async def test_response_cancellation_cleans_up_python_work(+ decision: bool, cancel_handler: bool+) -> None:+ config = (+ TaskClassifierConfig.decision(+ cutoff=0.5, candidates={"strong": "strong", "weak": "weak"}, evidence={}+ )+ if decision+ else TaskClassifierConfig(0.5)+ )+ algorithm = algorithms.llm_task_classifier(config=config)+ stream = algorithm.run_stream(+ {"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]},+ {+ "judge": ["judge"],+ "capable": ["strong"],+ "efficient": ["weak"],+ "any": ["strong", "weak"],+ },+ )+ step = await anext(stream)+ assert isinstance(step, Step.CallDecision if decision else Step.CallModel)+ started = asyncio.Event()+ cleaned_up = asyncio.Event()++ async def work() -> LlmResponse.Agg:+ try:+ started.set()+ await asyncio.Future()+ raise AssertionError("pending work completed")+ finally:+ await asyncio.sleep(0)+ cleaned_up.set()++ handler = asyncio.create_task(step.call.respond(work()))+ await asyncio.wait_for(started.wait(), timeout=2)+ if cancel_handler:+ handler.cancel()+ with pytest.raises(asyncio.CancelledError):+ await asyncio.wait_for(handler, timeout=2)+ outcome = await anext(stream)+ assert isinstance(outcome, Step.Done)+ assert outcome.outcome.selected_model_ids[0] == "strong"+ else:+ del stream+ gc.collect()+ await asyncio.wait_for(handler, timeout=2)+ assert cleaned_up.is_set()++[email protected]("failure", [False, True])+async def test_decision_response_and_failure_keep_their_routing_behavior(failure: bool) -> None:+ algorithm = algorithms.llm_task_classifier(+ config=TaskClassifierConfig.decision(+ cutoff=0.5, candidates={"strong": "strong", "weak": "weak"}, evidence={}+ )+ )+ stream = algorithm.run_stream(+ {"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]},+ {+ "judge": ["judge"],+ "capable": ["strong"],+ "efficient": ["weak"],+ "any": ["strong", "weak"],+ },+ )+ step = await anext(stream)+ assert isinstance(step, Step.CallDecision)++ async def answer() -> dict[str, object]:+ return {+ "id": None,+ "model": "judge",+ "answers": {+ "route": {+ "value": {+ "type": "choice",+ "data": {+ "selected": "no_advantage",+ "probabilities": {"advantage": 0.1, "no_advantage": 0.9},+ },+ },+ "provider_confidence": None,+ }+ },+ }++ if failure:+ await step.call.fail(ContextWindowExceededError("judge context overflow"))+ else:+ await step.call.respond(answer())+ outcome = await anext(stream)+ assert isinstance(outcome, Step.Done)+ assert outcome.outcome.selected_model_ids[0] == ("strong" if failure else "weak")+ with pytest.raises(LibsyError, match="already been completed"):+ await step.call.respond(answer())diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex e28ffc126..392db46ae 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -17,6 +17,7 @@ from switchyard.libsy import (EscalationClassifierConfig,LlmClassifierConfig,LlmResponse,+ ModelCall,OutcomeMetadata,RoutingOutcome,Step,@@ -56,6 +57,16 @@ class EchoClient:}+async def ready(+ response: LlmResponse.Agg | LlmResponse.Stream,+) -> LlmResponse.Agg | LlmResponse.Stream:+ return response+++async def aggregate(client: Any, request: dict[str, Any]) -> LlmResponse.Agg:+ return LlmResponse.Agg(await client.call(request))++async def run_algorithm(algorithm: Algorithm,clients: dict[str, Any] | None = None,@@ -64,26 +75,23 @@ async def run_algorithm(request: dict[str, Any] | None = None,headers: dict[str, str] | None = None,) -> tuple[str, dict[str, Any]]:+ async def serve(call: ModelCall) -> LlmResponse.Agg:+ for index, target in enumerate(call.models):+ candidate_request = {**call.request, "model": target}+ try:+ return await aggregate((clients or {})[target], candidate_request)+ except ContextWindowExceededError:+ if index + 1 == len(call.models):+ raise+ raise AssertionError("model call has no candidates")+runtime_models = models if models is not None else {"any": list((clients or {}).keys())}async for step in algorithm.run_stream(request or request_body(), runtime_models, headers=headers):match step:case Step.CallModel(call):- for index, target in enumerate(call.models):- candidate_request = {**call.request, "model": target}- client = (clients or {})[target]- try:- response = await client.call(candidate_request)- except ContextWindowExceededError as error:- if index + 1 == len(call.models):- call.fail(error)- except Exception as error:- call.fail(error)- break- else:- call.respond(LlmResponse.Agg(response))- break+ await call.respond(serve(call))case Step.Done(outcome):assert isinstance(outcome.metadata, OutcomeMetadata)assert UUID(outcome.metadata.outcome_id).version == 7@@ -167,7 +175,7 @@ async def test_routing_call_accepts_a_streamed_response() -> None:async for step in algorithm.run_stream(request_body(), models):match step:case Step.CallModel(call):- call.respond(LlmResponse.Stream(events()))+ await call.respond(ready(LlmResponse.Stream(events())))case Step.Done(done):outcome = done@@ -222,7 +230,7 @@ async def test_classifier_config_accepts_a_prompt_override() -> None:async for step in algorithm.run_stream(request_body(), models):match step:case Step.CallModel(call):- call.respond(LlmResponse.Agg(await judge.call(call.request)))+ await call.respond(aggregate(judge, call.request))case Step.Done(done):outcome = done@@ -505,7 +513,7 @@ async def test_escalation_falls_back_on_first_stream_context_overflow() -> None:match step:case Step.CallModel(call):calls.append(call.models)- call.respond(LlmResponse.Stream(overflow()))+ await call.respond(ready(LlmResponse.Stream(overflow())))case Step.Done(done):outcome = done