Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): cancel abandoned offloaded calls (#948)

nachiketb-nvidia50f635d74c72
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md
index 19a423c16..c2c948150 100644
--- a/crates/libsy-llm-client/README.md
+++ b/crates/libsy-llm-client/README.md
@@ -154,6 +154,12 @@ async fn stream(
runs. The terminal outcome either already contains the answer or supplies the selected model and
ordered fallbacks for the client to try. `ClientRouter::single` is the single-provider case:
+Routing-time calls run concurrently. If an algorithm drops one of its waiting
+call futures, the corresponding HTTP work is dropped while other calls continue.
+Ending or dropping the run drops all outstanding call handlers. A returned
+response stream stays alive until its consumer finishes or drops it. These are
+local cancellation guarantees; a provider may continue generation after disconnect.
+
```rust
use std::sync::Arc;
use switchyard_libsy::{Algorithm, RuntimeModels};
diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs
index dcb4a6f7d..ab8a3e9cb 100644
--- a/crates/libsy-llm-client/src/run.rs
+++ b/crates/libsy-llm-client/src/run.rs
@@ -163,21 +163,25 @@ async fn serve_decision(
observations: &Option<Arc<Mutex<Vec<RunObservation>>>>,
) -> Result<()> {
let client = clients.route_decision(&call.model);
- let started = Instant::now();
- let result = async { client?.call(call.request.clone()).await }.await;
- tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });
- if let Some(observations) = observations {
- observations
- .lock()
- .push(RunObservation::DecisionCall(ModelCallObservation {
- selected_model: call.model.clone(),
- is_success: result.is_ok(),
- duration: started.elapsed(),
- usage: result.as_ref().ok().map(|response| response.usage.clone()),
- }));
- }
- let result = result.map_err(|error| LibsyError::client_call(call.model.clone(), error));
- call.respond(result)
+ let model = call.model.clone();
+ let request = call.request.clone();
+ call.respond(async {
+ let started = Instant::now();
+ let result = async { client?.call(request).await }.await;
+ tracing::Span::current().record("outcome", if result.is_ok() { "ok" } else { "error" });
+ if let Some(observations) = observations {
+ observations
+ .lock()
+ .push(RunObservation::DecisionCall(ModelCallObservation {
+ selected_model: model.clone(),
+ is_success: result.is_ok(),
+ duration: started.elapsed(),
+ usage: result.as_ref().ok().map(|response| response.usage.clone()),
+ }));
+ }
+ result.map_err(|error| LibsyError::client_call(model, error))
+ })
+ .await
}
/// Emits completed routing calls after the outcome reveals whether one response became the answer.
@@ -223,24 +227,14 @@ async fn serve(
.push(RunObservation::LlmCall(observation));
}
};
- let target = call.models.first().ok_or(LibsyError::NoTargets)?;
- let request = clients.prepare_routing_request(call.request.clone(), target);
- match call_one(
- &clients,
- target,
- request,
- &call.algorithm,
- &observe,
- 0,
- call.models.len(),
- true,
- )
+ let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
+ let request = clients.prepare_routing_request(call.request.clone(), &target);
+ let algorithm = call.algorithm.clone();
+ let candidates = call.models.len();
+ call.respond(call_one(
+ &clients, &target, request, &algorithm, &observe, 0, candidates, true,
+ ))
.await
- {
- Ok(response) => call.respond(Ok(response)),
- Err(error) if call.recover_errors => call.respond(Err(error)),
- Err(error) => call.fail(error),
- }
}
/// Try candidates in order until one succeeds or a failure stops fallback.
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 30366dbb2..37a71f3db 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -1660,17 +1660,23 @@ async fn decision_calls_record_each_terminal_path_once() -> switchyard_libsy::Re
drop(call);
} else {
match mode {
- "reply" => call.respond(Ok(DecisionResponse {
- id: Some("decision-response".into()),
- model: Some("provider-model".into()),
- answers: Default::default(),
- usage: Usage {
- input_tokens: Some(42),
- output_tokens: Some(5),
- ..Usage::default()
- },
- }))?,
- "error" => call.respond(Err(test_error(LEAKED_CONTENT)))?,
+ "reply" => {
+ call.respond(std::future::ready(Ok(DecisionResponse {
+ id: Some("decision-response".into()),
+ model: Some("provider-model".into()),
+ answers: Default::default(),
+ usage: Usage {
+ input_tokens: Some(42),
+ output_tokens: Some(5),
+ ..Usage::default()
+ },
+ })))
+ .await?
+ }
+ "error" => {
+ call.respond(std::future::ready(Err(test_error(LEAKED_CONTENT))))
+ .await?
+ }
"fail" => assert!(call.fail(test_error(LEAKED_CONTENT)).is_err()),
"drop" => drop(call),
_ => unreachable!(),
@@ -1757,8 +1763,12 @@ async fn failed_call_records_metrics_without_error_details() -> switchyard_libsy
let mut saw_error_step = false;
while let Some(step) = stream.next().await {
match step {
- Ok(Step::CallModel(call)) => {
- call.respond(Err(test_error("synthetic upstream failure")))?;
+ Ok(Step::CallModel(mut call)) => {
+ call.recover_errors = true;
+ call.respond(std::future::ready(Err(test_error(
+ "synthetic upstream failure",
+ ))))
+ .await?;
}
Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),
Ok(Step::Done(_)) => {
@@ -2131,11 +2141,12 @@ async fn in_flight_gauge_reads_a_run_parked_on_an_unanswered_routing_call()
"the run counter must not report a run that has not resolved"
);
- call.respond(Ok(Response {
+ call.respond(std::future::ready(Ok(Response {
llm_response: LlmResponse::Agg(text_response(Some(MODEL.to_string()), "answer")),
metadata: None,
upstream_headers: http::HeaderMap::new(),
- }))?;
+ })))
+ .await?;
while stream.next().await.is_some() {}
let snapshots = flushed_metrics(exporter, provider);
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index a7df749ee..0be56eb61 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -41,6 +41,28 @@ drives the stream and performs the terminal answer call, retries, and fallback o
The provider-neutral [`Request`], [`Response`], [`Usage`], and [`LlmResponse`]
contracts come from `switchyard-protocol`.
+## Concurrent calls and cancellation
+
+An algorithm can poll several `Driver::call_model` or `Driver::call_decision` futures
+with `tokio::join!`, `tokio::select!`, or `FuturesUnordered`. Each call has its own
+reply channel. Dropping a waiting future cancels only that call; the algorithm can
+continue making other calls.
+
+Hosts pass the work itself to `call.respond(work).await`. Awaiting the provider
+before calling `respond` prevents libsy from cancelling that work. For a response
+that is already available, use `call.respond(std::future::ready(result)).await`.
+Model-call errors stop the host run unless `call.recover_errors` is enabled.
+Decision-call errors are returned to the algorithm.
+
+`drive` polls host handlers concurrently and drops outstanding handlers when the
+algorithm finishes or the run is dropped. Custom `run_stream` consumers must also
+poll handlers concurrently and drop them when the run ends. Separately spawned
+tasks need explicit abort and cleanup; dropping a task handle does not stop its work.
+
+Cancellation drops the supplied work future. Once a response stream is delivered,
+its consumer controls its lifetime. Whether the provider stops generation depends
+on the transport and provider.
+
[`Request`]: switchyard_protocol::Request
[`Response`]: switchyard_protocol::Response
[`Usage`]: switchyard_protocol::Usage
diff --git a/crates/libsy/src/algorithms/advisor_gate/tests.rs b/crates/libsy/src/algorithms/advisor_gate/tests.rs
index 8bf901595..b59e549e8 100644
--- a/crates/libsy/src/algorithms/advisor_gate/tests.rs
+++ b/crates/libsy/src/algorithms/advisor_gate/tests.rs
@@ -356,7 +356,7 @@ async fn calls_preserve_candidates_and_attribute_the_serving_executor() {
};
let mut response = reply(text);
response.set_served_model(&served);
- call.respond(Ok(response))
+ call.respond(std::future::ready(Ok(response))).await
}
})
.await
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs
index 7e3d15b83..cced4d9fb 100644
--- a/crates/libsy/src/algorithms/escalation.rs
+++ b/crates/libsy/src/algorithms/escalation.rs
@@ -634,7 +634,7 @@ mod tests {
.serve(target.clone(), request)
.await
.map_err(|source| LibsyError::client_call(target, source));
- call.respond(response)
+ call.respond(std::future::ready(response)).await
}
},
)
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index e45d73ff7..a95683790 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -1408,10 +1408,12 @@ mod tests {
}
}
if name == "provider error" {
- return call.respond(Err(LibsyError::client_call(
- "judge",
- LlmClientError::General("private provider body".into()),
- )));
+ return call
+ .respond(std::future::ready(Err(LibsyError::client_call(
+ "judge",
+ LlmClientError::General("private provider body".into()),
+ ))))
+ .await;
}
if name == "dropped reply" {
drop(call);
@@ -1441,12 +1443,13 @@ mod tests {
},
)])
};
- call.respond(Ok(DecisionResponse {
+ call.respond(std::future::ready(Ok(DecisionResponse {
id: None,
model: Some("provider-judge".into()),
answers,
usage: Default::default(),
- }))
+ })))
+ .await
}
};
let models = Arc::new(RuntimeModels::new(runtime_models()));
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 70ff7ef1e..2ca10cc82 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -588,7 +588,7 @@ mod tests {
let serve = async {
if let Some(Ok(Step::CallModel(call))) = steps.next().await {
- let _ = call.respond(reply);
+ let _ = call.respond(std::future::ready(reply)).await;
}
};
let (classification, ()) =
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 133f8fbe4..6e45c1267 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -96,8 +96,8 @@ enum Scope {
/// An offloaded model call, surfaced inside [`Step::CallModel`].
///
-/// The host reads the public fields, performs (or delegates) the model call, and fulfills it
-/// with [`respond`](Self::respond) — unblocking the algorithm's [`Driver::call_model`] on the
+/// The host reads the public fields and passes its model-call future to
+/// [`respond`](Self::respond), unblocking the algorithm's [`Driver::call_model`] on the
/// other side. `switchyard-llm-client`'s `run` is the ready-made consumer that does this for
/// you.
///
@@ -121,10 +121,26 @@ pub struct CallModel {
}
impl CallModel {
- /// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to
- /// propagate a failed model call back to the algorithm. Consumes the promise: it
- /// can only be fulfilled once.
- pub fn respond(mut self, result: Result<Response>) -> Result<()> {
+ /// Run host work until it completes or the algorithm stops waiting for this call.
+ /// Dropping the waiting [`Driver::call_model`] future drops `work` and returns `Ok(())`.
+ /// Errors stop the host's run unless [`Self::recover_errors`] is enabled.
+ /// Use `std::future::ready(result)` for an already available response.
+ /// A delivered response stream owns its subsequent lifetime.
+ pub async fn respond(mut self, work: impl Future<Output = Result<Response>>) -> Result<()> {
+ let reply = self.reply.as_mut().ok_or(DriverError::ResponseDropped)?;
+ let result = tokio::select! {
+ biased;
+ _ = reply.closed() => return Ok(()),
+ result = work => result,
+ };
+ if self.reply.as_ref().is_some_and(oneshot::Sender::is_closed) {
+ return Ok(());
+ }
+ if !self.recover_errors
+ && let Err(error) = result
+ {
+ return self.fail(error);
+ }
self.record(result.is_ok());
if let Ok(response) = &result {
observability::record_llm_response(response, &self.span);
@@ -133,7 +149,8 @@ impl CallModel {
.take()
.ok_or(DriverError::ResponseDropped)?
.send(result)
- .map_err(|_| DriverError::ResponseDropped.into())
+ .ok();
+ Ok(())
}
/// Record a failed call and return its error to stop [`drive`].
@@ -183,8 +200,19 @@ pub struct CallDecision {
}
impl CallDecision {
- /// Return a response or provider error to the algorithm so it can continue or fall back.
- pub fn respond(mut self, result: Result<DecisionResponse>) -> Result<()> {
+ /// Run host work and return its response or error to the algorithm.
+ /// Dropping the waiting [`Driver::call_decision`] future drops `work` and returns `Ok(())`.
+ /// Use `std::future::ready(result)` for an already available response.
+ pub async fn respond(
+ mut self,
+ work: impl Future<Output = Result<DecisionResponse>>,
+ ) -> Result<()> {
+ let reply = self.reply.as_mut().ok_or(DriverError::ResponseDropped)?;
+ let result = tokio::select! {
+ biased;
+ _ = reply.closed() => return Ok(()),
+ result = work => result,
+ };
self.record(result.is_ok());
if let Ok(response) = &result {
observability::record_decision_response(response, &self.span);
@@ -193,7 +221,8 @@ impl CallDecision {
.take()
.ok_or(DriverError::ResponseDropped)?
.send(result)
- .map_err(|_| DriverError::ResponseDropped.into())
+ .ok();
+ Ok(())
}
/// Returning this error from the host handler aborts [`drive`].
@@ -862,11 +891,11 @@ mod tests {
calls
.remove("second")
.ok_or_else(|| test_error("missing second call"))?
- .respond(Ok(reply("second response")))?;
+ .respond(std::future::ready(Ok(reply("second response")))).await?;
calls
.remove("first")
.ok_or_else(|| test_error("missing first call"))?
- .respond(Ok(reply("first response")))?;
+ .respond(std::future::ready(Ok(reply("first response")))).await?;
let first_response = first
.await
@@ -998,7 +1027,7 @@ mod tests {
async move {
barrier.wait().await;
let call = match call {
- Call::Model(call) => return call.respond(Ok(reply("llm reply"))),
+ Call::Model(call) => return call.respond(std::future::ready(Ok(reply("llm reply")))).await,
Call::Decision(call) => *call,
};
assert_eq!(call.algorithm, "mixed");
@@ -1007,10 +1036,10 @@ mod tests {
assert_eq!(call.request.context, decision_request().context);
match mode {
"mock" => serve_decision(call).await,
- "reply" => call.respond(Ok(decision_response())),
- "error" => call.respond(Err(LibsyError::AlgorithmError {
+ "reply" => call.respond(std::future::ready(Ok(decision_response()))).await,
+ "error" => call.respond(std::future::ready(Err(LibsyError::AlgorithmError {
message: "provider failed".into(),
- })),
+ }))).await,
"drop" => {
drop(call);
Ok(())
@@ -1036,10 +1065,7 @@ mod tests {
return Err(test_error("expected a decision call"));
};
drop(pending);
- assert!(matches!(
- call.respond(Ok(decision_response())),
- Err(LibsyError::Driver(DriverError::ResponseDropped))
- ));
+ call.respond(async { panic!("cancelled work must not be polled") }).await?;
drop(steps);
assert!(matches!(
driver.call_decision(decision_request(), "decision".into()).await,
@@ -1153,14 +1179,15 @@ mod tests {
saw_call = true;
assert_eq!(call.models, vec![ModelId::from("offload/model")]);
// Fulfilling the promise is the "real" model call the caller makes.
- call.respond(Ok(Response {
+ call.respond(std::future::ready(Ok(Response {
llm_response: LlmResponse::Agg(text_response(
None,
"fulfilled".to_string(),
)),
metadata: None,
upstream_headers: http::HeaderMap::new(),
- }))?;
+ })))
+ .await?;
}
Step::Done(outcome) => {
let metadata = outcome
@@ -1263,8 +1290,12 @@ mod tests {
while let Some(step) = stream.next().await {
match step {
Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),
- Ok(Step::CallModel(call)) => {
- call.respond(Err(test_error("upstream model call failed")))?;
+ Ok(Step::CallModel(mut call)) => {
+ call.recover_errors = true;
+ call.respond(std::future::ready(Err(test_error(
+ "upstream model call failed",
+ ))))
+ .await?;
}
Ok(Step::Done(..)) => {
return Err(test_error(
diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rs
index b62667cb3..d256a592e 100644
--- a/crates/libsy/src/core/testing.rs
+++ b/crates/libsy/src/core/testing.rs
@@ -101,23 +101,27 @@ pub(crate) async fn serve_decision(call: CallDecision) -> Result<()> {
answers: Default::default(),
usage: Default::default(),
};
- call.respond(Ok(response))
+ call.respond(std::future::ready(Ok(response))).await
}
/// Serve one call and fulfill its promise, mapping failures the way a host does so
/// error-shape assertions match production.
async fn fulfill(serve: Arc<impl Serve>, call: Call) -> Result<()> {
- let call = match call {
+ let mut call = match call {
Call::Model(call) => *call,
Call::Decision(call) => return serve_decision(*call).await,
};
let request = call.request.clone();
let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;
- let result = serve
- .serve(target.clone(), request)
- .await
- .map_err(|source| LibsyError::client_call(target, source));
- call.respond(result)
+ // Test clients return failures to routing policies, including policies under test.
+ call.recover_errors = true;
+ call.respond(async {
+ serve
+ .serve(target.clone(), request)
+ .await
+ .map_err(|source| LibsyError::client_call(target, source))
+ })
+ .await
}
/// Answers with the selected model name as the completion — what most routing tests need,
diff --git a/crates/libsy/tests/cancellation.rs b/crates/libsy/tests/cancellation.rs
new file mode 100644
index 000000000..242404ad3
--- /dev/null
+++ b/crates/libsy/tests/cancellation.rs
@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use switchyard_libsy::{Algorithm, Call, Driver, Result, RoutingOutcome, RuntimeModels};
+use switchyard_protocol::{
+ DecisionRequest, LlmResponse, Request, Response, text_request, text_response,
+};
+use tokio::sync::Notify;
+
+struct Abandon {
+ decision: bool,
+ started: Arc<Notify>,
+ dropped: Arc<Notify>,
+}
+
+#[async_trait]
+impl Algorithm for Abandon {
+ fn name(&self) -> &str {
+ "abandon"
+ }
+
+ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<RoutingOutcome> {
+ {
+ let call = async {
+ if self.decision {
+ driver
+ .call_decision(
+ DecisionRequest {
+ model: None,
+ context: serde_json::json!({}),
+ questions: Default::default(),
+ },
+ "abandoned".into(),
+ )
+ .await
+ .map(|_| ())
+ } else {
+ driver
+ .call_model(request.clone(), vec!["abandoned".into()])
+ .await
+ .map(|_| ())
+ }
+ };
+ tokio::pin!(call);
+ tokio::select! {
+ _ = self.started.notified() => {},
+ _ = &mut call => panic!("abandoned work finished"),
+ }
+ }
+ // The next call cannot begin until the host drops the abandoned work.
+ self.dropped.notified().await;
+ let response = driver
+ .call_model(request.clone(), vec!["continuation".into()])
+ .await?;
+ Ok(RoutingOutcome::answered(
+ "continuation".into(),
+ request,
+ response,
+ ))
+ }
+}
+
+struct DropNotify(Arc<Notify>);
+impl Drop for DropNotify {
+ fn drop(&mut self) {
+ self.0.notify_one();
+ }
+}
+
+async fn pending<T>(started: Arc<Notify>, dropped: Arc<Notify>) -> Result<T> {
+ let _guard = DropNotify(dropped);
+ started.notify_one();
+ std::future::pending().await
+}
+
+fn request() -> Request {
+ Request {
+ llm_request: text_request(None, "hello"),
+ raw_request: None,
+ metadata: None,
+ }
+}
+
+fn response() -> Response {
+ Response {
+ llm_response: LlmResponse::Agg(text_response(None, "continued")),
+ metadata: None,
+ upstream_headers: Default::default(),
+ }
+}
+
+#[tokio::test]
+async fn abandoning_model_or_decision_work_cancels_only_that_call() {
+ for decision in [false, true] {
+ let started = Arc::new(Notify::new());
+ let dropped = Arc::new(Notify::new());
+ let algorithm = Arc::new(Abandon {
+ decision,
+ started: started.clone(),
+ dropped: dropped.clone(),
+ });
+ let outcome = tokio::time::timeout(
+ Duration::from_secs(1),
+ switchyard_libsy::drive(
+ algorithm,
+ request(),
+ Arc::new(RuntimeModels::default()),
+ move |call| {
+ let started = started.clone();
+ let dropped = dropped.clone();
+ async move {
+ match call {
+ Call::Model(call) if call.models[0] == "continuation" => {
+ call.respond(std::future::ready(Ok(response()))).await
+ }
+ Call::Model(call) => call.respond(pending(started, dropped)).await,
+ Call::Decision(call) => call.respond(pending(started, dropped)).await,
+ }
+ }
+ },
+ ),
+ )
+ .await
+ .expect("cancelled call blocked continuation")
+ .expect("cancelled call failed the run");
+ assert_eq!(outcome.selected_model_id().unwrap(), "continuation");
+ }
+}
diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rs
index bd66d230d..f1b338cf9 100644
--- a/crates/prefill-router/tests/unit/algorithm.rs
+++ b/crates/prefill-router/tests/unit/algorithm.rs
@@ -72,7 +72,7 @@ async fn serve_decision(call: libsy::CallDecision) -> libsy::Result<()> {
answers: Default::default(),
usage: Default::default(),
};
- call.respond(Ok(response))
+ call.respond(std::future::ready(Ok(response))).await
}
async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<String> {
@@ -85,13 +85,14 @@ async fn selected(route: Arc<dyn Algorithm>, request: Request) -> libsy::Result<
Call::Model(call) => *call,
Call::Decision(call) => return serve_decision(*call).await,
};
- call.respond(Ok(switchyard_protocol::Response {
+ call.respond(std::future::ready(Ok(switchyard_protocol::Response {
llm_response: switchyard_protocol::LlmResponse::Agg(
switchyard_protocol::text_response(None, "unused"),
),
metadata: None,
upstream_headers: Default::default(),
- }))
+ })))
+ .await
},
)
.await?;
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 598c830f2..aa98c91db 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -542,8 +542,22 @@ impl PyModelCall {
self.models.clone()
}
- /// Fulfill this call with a normalized aggregate or streamed response.
- fn respond(&mut self, py: Python<'_>, response: PyRef<'_, PyLlmResponse>) -> PyResult<()> {
+ /// Await a normalized response, cancelling Python work if the algorithm stops waiting.
+ fn respond<'py>(
+ slf: PyRef<'py, Self>,
+ py: Python<'py>,
+ work: &Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ py.import("switchyard_rust._libsy_async")?
+ .getattr("respond")?
+ .call1((slf, work))
+ }
+
+ fn _respond<'py>(
+ &mut self,
+ py: Python<'py>,
+ work: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
let model = self
.inner
.as_ref()
@@ -554,36 +568,53 @@ impl PyModelCall {
.as_ref()
.map(ModelId::new)
.ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;
- let llm_response = response.to_core(py, model)?;
+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
+ let work = pyo3_async_runtimes::into_future_with_locals(&locals, work)?;
let call = self.take()?;
let metadata = call.request.metadata.clone();
- call.respond(Ok(Response {
- llm_response,
- metadata,
- upstream_headers: HeaderMap::new(),
- }))
- .map_err(py_libsy_error)
+ pyo3_async_runtimes::tokio::future_into_py(py, async move {
+ call.respond(async move {
+ let result = work.await;
+ Python::attach(|py| {
+ let response = result.map_err(|error| {
+ RustLibsyError::client_call(
+ model.clone(),
+ python_client_error(py, error, &model),
+ )
+ })?;
+ let response = response
+ .bind(py)
+ .extract::<PyRef<'_, PyLlmResponse>>()
+ .map_err(|error| ffi_error(error.into()))
+ .map_err(|source| RustLibsyError::client_call(model.clone(), source))?;
+ let llm_response = response
+ .to_core(py, model.clone())
+ .map_err(ffi_error)
+ .map_err(|source| RustLibsyError::client_call(model, source))?;
+ Ok(Response {
+ llm_response,
+ metadata,
+ upstream_headers: HeaderMap::new(),
+ })
+ })
+ })
+ .await
+ .map_err(py_libsy_error)
+ })
}
- /// Fulfill this call with a Python client failure.
- fn fail(&mut self, error: &Bound<'_, PyAny>) -> PyResult<()> {
+ /// Return a Python client failure through the same async reply path.
+ fn fail<'py>(
+ slf: PyRef<'py, Self>,
+ py: Python<'py>,
+ error: &Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
if !error.is_instance_of::<PyBaseException>() {
return Err(PyTypeError::new_err("error must derive from BaseException"));
}
- let target = self
- .inner
- .as_ref()
- .ok_or_else(|| py_libsy_error("model call has already been completed"))?
- .request
- .llm_request
- .model
- .as_ref()
- .map(ModelId::new)
- .ok_or_else(|| py_libsy_error("model call request is missing its selected model"))?;
- let call = self.take()?;
- let source = python_client_error(error.py(), PyErr::from_value(error.clone()), &target);
- call.respond(Err(RustLibsyError::client_call(target, source)))
- .map_err(py_libsy_error)
+ py.import("switchyard_rust._libsy_async")?
+ .getattr("fail")?
+ .call1((slf, error))
}
}
@@ -632,22 +663,58 @@ impl PyDecisionCall {
&self.model
}
- /// Return a provider-neutral decision response to the waiting algorithm.
- fn respond(&mut self, response: &Bound<'_, PyAny>) -> PyResult<()> {
- let response: DecisionResponse = from_python(response)?;
- self.take()?.respond(Ok(response)).map_err(py_libsy_error)
+ /// Await a decision response, cancelling Python work if the algorithm stops waiting.
+ fn respond<'py>(
+ slf: PyRef<'py, Self>,
+ py: Python<'py>,
+ work: &Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ py.import("switchyard_rust._libsy_async")?
+ .getattr("respond")?
+ .call1((slf, work))
+ }
+
+ fn _respond<'py>(
+ &mut self,
+ py: Python<'py>,
+ work: Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
+ let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
+ let work = pyo3_async_runtimes::into_future_with_locals(&locals, work)?;
+ let call = self.take()?;
+ let model = call.model.clone();
+ pyo3_async_runtimes::tokio::future_into_py(py, async move {
+ call.respond(async move {
+ let result = work.await;
+ Python::attach(|py| {
+ let response = result.map_err(|error| {
+ RustLibsyError::client_call(
+ model.clone(),
+ python_client_error(py, error, &model),
+ )
+ })?;
+ from_python::<DecisionResponse>(response.bind(py))
+ .map_err(ffi_error)
+ .map_err(|source| RustLibsyError::client_call(model, source))
+ })
+ })
+ .await
+ .map_err(py_libsy_error)
+ })
}
/// Return a Python client failure so the algorithm can apply its fallback policy.
- fn fail(&mut self, error: &Bound<'_, PyAny>) -> PyResult<()> {
+ fn fail<'py>(
+ slf: PyRef<'py, Self>,
+ py: Python<'py>,
+ error: &Bound<'py, PyAny>,
+ ) -> PyResult<Bound<'py, PyAny>> {
if !error.is_instance_of::<PyBaseException>() {
return Err(PyTypeError::new_err("error must derive from BaseException"));
}
- let source = python_client_error(error.py(), PyErr::from_value(error.clone()), &self.model);
- let model = self.model.clone();
- self.take()?
- .respond(Err(RustLibsyError::client_call(model, source)))
- .map_err(py_libsy_error)
+ py.import("switchyard_rust._libsy_async")?
+ .getattr("fail")?
+ .call1((slf, error))
}
}
diff --git a/docs/getting_started.md b/docs/getting_started.md
index 5d524765e..41c1a0d23 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -236,6 +236,42 @@ response when routing already produced the answer. Otherwise the host makes the
call from that outcome. Serving these calls yourself is what lets libsy embed in a host that
already owns its HTTP stack, retries, and credentials.
+Pass each call's work to `respond` before awaiting it. In Rust:
+
+```rust
+use switchyard_libsy::{CallModel, LibsyError, Result};
+use switchyard_protocol::RoutedLlmClient;
+
+async fn serve(call: CallModel, client: &dyn RoutedLlmClient) -> Result<()> {
+ let request = call.request.clone();
+ let model = call.models[0].clone();
+ call.respond(async move {
+ client.call(request).await.map_err(|error| LibsyError::client_call(model, error))
+ }).await
+}
+```
+
+In Python, the client coroutine returns `LlmResponse.Agg` or `LlmResponse.Stream`:
+
+```python
+await call.respond(client.call(call.request, call.models[0]))
+```
+
+`respond` now accepts a future or awaitable instead of a completed response. Move
+the provider await into this method. Python `call.fail(error)` is also awaitable.
+Decision calls use the same pattern with their existing decision response type.
+
+When an algorithm stops waiting for a call, `respond` cancels that work and
+returns normally. Python waits for task cleanup before returning. For concurrent
+algorithms, schedule each Python handler with `asyncio.create_task` and cancel and
+await remaining handlers when the run ends; see the complete
+[Python example](../examples/libsy.py). Awaiting handlers one at a time inside the
+step loop serializes their execution. Rust hosts can use `libsy::drive` to manage
+concurrent handlers and their lifetime.
+
+After a response stream is delivered, its consumer controls its lifetime.
+Cancellation cannot guarantee that an upstream provider stops generating or billing.
+
Successful runs also include `OutcomeMetadata`: a unique `outcome_id`, the algorithm
name, and optional JSON evidence. In Python, read `outcome.metadata.outcome_id`,
`outcome.metadata.algorithm`, and `outcome.metadata.evidence` after checking that
diff --git a/docs/reference/opentelemetry.md b/docs/reference/opentelemetry.md
index 2d798b41d..34c8e975b 100644
--- a/docs/reference/opentelemetry.md
+++ b/docs/reference/opentelemetry.md
@@ -173,7 +173,7 @@ Advisor Gate instruments use the prefix `switchyard.advisor_gate.`:
### Timing and streaming
- `libsy.run` may finish before the answer call. Nested algorithms have separate run spans.
-- Decision calls record `error` when failed or dropped without a reply, including cancellation. A returned response records its outcome even if the waiting algorithm has already gone away. These metrics work with any host and do not require a client observer.
+- Decision calls record `error` when failed or dropped without a reply, including cancellation. Cancelled calls do not poll further host work. A response completed during a cancellation race may still record its result. These metrics work with any host and do not require a client observer.
- `libsy.llm_call` and routing-time call metrics end when the host fulfills the offloaded call. They include host queueing. The LLM client driver buffers routing streams before fulfilling the call. The span's `input_tokens`, `output_tokens`, `total_tokens`, and `reasoning_tokens` fields are buffered-response only.
- For terminal answer candidates, `switchyard.llm_calls` and `switchyard.llm_call_duration_ms` record when the response stream ends or is dropped. Duration includes that candidate's retries and stream consumption. Stream errors and unfinished drops record `outcome=error`; a terminal message permits a successful drop.
- `libsy.client_call` remains open while its stream is consumed. IDs, usage, and finish reasons update from normalized events. An unfinished stream dropped by its consumer records `cancelled`; a stream error records `error`.
diff --git a/examples/libsy.py b/examples/libsy.py
index 42da987df..719341ad2 100644
--- a/examples/libsy.py
+++ b/examples/libsy.py
@@ -7,7 +7,7 @@
import asyncio
from collections.abc import AsyncIterator, Mapping
-from switchyard.libsy import LlmResponse, Step, algorithms
+from switchyard.libsy import LlmResponse, RoutingOutcome, Step, algorithms
class EchoClient:
@@ -62,22 +62,33 @@ async def main() -> None:
seed=42,
)
- async for step in algorithm.run_stream(request, {"any": ["fast", "quality"]}):
- match step:
- case Step.CallModel(call):
- call.respond(await client.call(call.request, call.models[0]))
- case Step.Done(outcome):
- print("Decision:", outcome.selected_model_ids[0])
- response = outcome.response or await client.call(
- outcome.request,
- outcome.selected_model_ids[0],
- )
- match response:
- case LlmResponse.Agg(aggregate_response):
- print("Response:", aggregate_response)
- case LlmResponse.Stream(response_stream):
- async for event in response_stream:
- print("Response event:", event)
+ handlers: list[asyncio.Task[None]] = []
+ outcome: RoutingOutcome | None = None
+ try:
+ async for step in algorithm.run_stream(request, {"any": ["fast", "quality"]}):
+ match step:
+ case Step.CallModel(call):
+ handlers.append(
+ asyncio.create_task(call.respond(client.call(call.request, call.models[0])))
+ )
+ case Step.Done(done):
+ outcome = done
+ break
+ finally:
+ for handler in handlers:
+ handler.cancel()
+ await asyncio.gather(*handlers, return_exceptions=True)
+
+ if outcome is None:
+ raise RuntimeError("Algorithm ended without a routing outcome")
+ print("Decision:", outcome.selected_model_ids[0])
+ response = outcome.response or await client.call(outcome.request, outcome.selected_model_ids[0])
+ match response:
+ case LlmResponse.Agg(aggregate_response):
+ print("Response:", aggregate_response)
+ case LlmResponse.Stream(response_stream):
+ async for event in response_stream:
+ print("Response event:", event)
if __name__ == "__main__":
diff --git a/switchyard_rust/_libsy_async.py b/switchyard_rust/_libsy_async.py
new file mode 100644
index 000000000..07ea10194
--- /dev/null
+++ b/switchyard_rust/_libsy_async.py
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Keep Python call tasks alive only while their libsy reply is needed."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable
+from typing import Any, Protocol
+
+
+class _Call(Protocol):
+ def _respond(self, task: asyncio.Future[Any]) -> Awaitable[None]: ...
+
+
+async def respond(call: _Call, work: Awaitable[Any]) -> None:
+ task = asyncio.ensure_future(work)
+ try:
+ await call._respond(task)
+ finally:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+
+
+async def fail(call: _Call, error: BaseException) -> None:
+ async def raise_error() -> None:
+ raise error
+
+ await respond(call, raise_error())
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index af56c7df8..15ff7fc49 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -36,7 +36,7 @@ _EXPORTS = frozenset(
)
if TYPE_CHECKING:
- from collections.abc import AsyncIterator, Sequence
+ from collections.abc import AsyncIterator, Awaitable, Sequence
from typing import ClassVar, Literal, final
class LibsyError(RuntimeError): ...
@@ -130,9 +130,9 @@ if TYPE_CHECKING:
@property
def models(self) -> list[str]: ...
- def respond(self, response: LlmResponse.Agg | LlmResponse.Stream) -> None: ...
+ async def respond(self, work: Awaitable[LlmResponse.Agg | LlmResponse.Stream]) -> None: ...
- def fail(self, error: BaseException) -> None: ...
+ async def fail(self, error: BaseException) -> None: ...
@final
class DecisionCall:
@@ -145,9 +145,9 @@ if TYPE_CHECKING:
@property
def model(self) -> str: ...
- def respond(self, response: Mapping[str, object]) -> None: ...
+ async def respond(self, work: Awaitable[Mapping[str, object]]) -> None: ...
- def fail(self, error: BaseException) -> None: ...
+ async def fail(self, error: BaseException) -> None: ...
@final
class OutcomeMetadata:
diff --git a/tests/test_libsy_cancellation.py b/tests/test_libsy_cancellation.py
new file mode 100644
index 000000000..150f47b56
--- /dev/null
+++ b/tests/test_libsy_cancellation.py
@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Cancellation crosses the Python/Rust boundary and awaits Python cleanup."""
+
+import asyncio
+import gc
+
+import pytest
+
+from switchyard.libsy import (
+ ContextWindowExceededError,
+ LibsyError,
+ LlmResponse,
+ Step,
+ TaskClassifierConfig,
+ algorithms,
+)
+
+
[email protected]("decision", [False, True])
[email protected]("cancel_handler", [False, True])
+async def test_response_cancellation_cleans_up_python_work(
+ decision: bool, cancel_handler: bool
+) -> None:
+ config = (
+ TaskClassifierConfig.decision(
+ cutoff=0.5, candidates={"strong": "strong", "weak": "weak"}, evidence={}
+ )
+ if decision
+ else TaskClassifierConfig(0.5)
+ )
+ algorithm = algorithms.llm_task_classifier(config=config)
+ stream = algorithm.run_stream(
+ {"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]},
+ {
+ "judge": ["judge"],
+ "capable": ["strong"],
+ "efficient": ["weak"],
+ "any": ["strong", "weak"],
+ },
+ )
+ step = await anext(stream)
+ assert isinstance(step, Step.CallDecision if decision else Step.CallModel)
+ started = asyncio.Event()
+ cleaned_up = asyncio.Event()
+
+ async def work() -> LlmResponse.Agg:
+ try:
+ started.set()
+ await asyncio.Future()
+ raise AssertionError("pending work completed")
+ finally:
+ await asyncio.sleep(0)
+ cleaned_up.set()
+
+ handler = asyncio.create_task(step.call.respond(work()))
+ await asyncio.wait_for(started.wait(), timeout=2)
+ if cancel_handler:
+ handler.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await asyncio.wait_for(handler, timeout=2)
+ outcome = await anext(stream)
+ assert isinstance(outcome, Step.Done)
+ assert outcome.outcome.selected_model_ids[0] == "strong"
+ else:
+ del stream
+ gc.collect()
+ await asyncio.wait_for(handler, timeout=2)
+ assert cleaned_up.is_set()
+
+
[email protected]("failure", [False, True])
+async def test_decision_response_and_failure_keep_their_routing_behavior(failure: bool) -> None:
+ algorithm = algorithms.llm_task_classifier(
+ config=TaskClassifierConfig.decision(
+ cutoff=0.5, candidates={"strong": "strong", "weak": "weak"}, evidence={}
+ )
+ )
+ stream = algorithm.run_stream(
+ {"messages": [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]},
+ {
+ "judge": ["judge"],
+ "capable": ["strong"],
+ "efficient": ["weak"],
+ "any": ["strong", "weak"],
+ },
+ )
+ step = await anext(stream)
+ assert isinstance(step, Step.CallDecision)
+
+ async def answer() -> dict[str, object]:
+ return {
+ "id": None,
+ "model": "judge",
+ "answers": {
+ "route": {
+ "value": {
+ "type": "choice",
+ "data": {
+ "selected": "no_advantage",
+ "probabilities": {"advantage": 0.1, "no_advantage": 0.9},
+ },
+ },
+ "provider_confidence": None,
+ }
+ },
+ }
+
+ if failure:
+ await step.call.fail(ContextWindowExceededError("judge context overflow"))
+ else:
+ await step.call.respond(answer())
+ outcome = await anext(stream)
+ assert isinstance(outcome, Step.Done)
+ assert outcome.outcome.selected_model_ids[0] == ("strong" if failure else "weak")
+ with pytest.raises(LibsyError, match="already been completed"):
+ await step.call.respond(answer())
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index e28ffc126..392db46ae 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -17,6 +17,7 @@ from switchyard.libsy import (
EscalationClassifierConfig,
LlmClassifierConfig,
LlmResponse,
+ ModelCall,
OutcomeMetadata,
RoutingOutcome,
Step,
@@ -56,6 +57,16 @@ class EchoClient:
}
+async def ready(
+ response: LlmResponse.Agg | LlmResponse.Stream,
+) -> LlmResponse.Agg | LlmResponse.Stream:
+ return response
+
+
+async def aggregate(client: Any, request: dict[str, Any]) -> LlmResponse.Agg:
+ return LlmResponse.Agg(await client.call(request))
+
+
async def run_algorithm(
algorithm: Algorithm,
clients: dict[str, Any] | None = None,
@@ -64,26 +75,23 @@ async def run_algorithm(
request: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
) -> tuple[str, dict[str, Any]]:
+ async def serve(call: ModelCall) -> LlmResponse.Agg:
+ for index, target in enumerate(call.models):
+ candidate_request = {**call.request, "model": target}
+ try:
+ return await aggregate((clients or {})[target], candidate_request)
+ except ContextWindowExceededError:
+ if index + 1 == len(call.models):
+ raise
+ raise AssertionError("model call has no candidates")
+
runtime_models = models if models is not None else {"any": list((clients or {}).keys())}
async for step in algorithm.run_stream(
request or request_body(), runtime_models, headers=headers
):
match step:
case Step.CallModel(call):
- for index, target in enumerate(call.models):
- candidate_request = {**call.request, "model": target}
- client = (clients or {})[target]
- try:
- response = await client.call(candidate_request)
- except ContextWindowExceededError as error:
- if index + 1 == len(call.models):
- call.fail(error)
- except Exception as error:
- call.fail(error)
- break
- else:
- call.respond(LlmResponse.Agg(response))
- break
+ await call.respond(serve(call))
case Step.Done(outcome):
assert isinstance(outcome.metadata, OutcomeMetadata)
assert UUID(outcome.metadata.outcome_id).version == 7
@@ -167,7 +175,7 @@ async def test_routing_call_accepts_a_streamed_response() -> None:
async for step in algorithm.run_stream(request_body(), models):
match step:
case Step.CallModel(call):
- call.respond(LlmResponse.Stream(events()))
+ await call.respond(ready(LlmResponse.Stream(events())))
case Step.Done(done):
outcome = done
@@ -222,7 +230,7 @@ async def test_classifier_config_accepts_a_prompt_override() -> None:
async for step in algorithm.run_stream(request_body(), models):
match step:
case Step.CallModel(call):
- call.respond(LlmResponse.Agg(await judge.call(call.request)))
+ await call.respond(aggregate(judge, call.request))
case Step.Done(done):
outcome = done
@@ -505,7 +513,7 @@ async def test_escalation_falls_back_on_first_stream_context_overflow() -> None:
match step:
case Step.CallModel(call):
calls.append(call.models)
- call.respond(LlmResponse.Stream(overflow()))
+ await call.respond(ready(LlmResponse.Stream(overflow())))
case Step.Done(done):
outcome = done