Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): propagate errors and cancel the algorithm task in `run_stream` (#114)
Greg Clarkf91b61aa4964
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 621fb980b..6313ef3ff 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -209,6 +209,15 @@ pub enum Step {ReturnToAgent(Box<Response>),}+/// Abort guard+struct AbortOnDrop(tokio::task::AbortHandle);++impl Drop for AbortOnDrop {+ fn drop(&mut self) {+ self.0.abort();+ }+}+/// A named routing target: a `semantic_name` an algorithm routes by, and an optional/// [`RoutedLlmClient`] to serve its calls. An algorithm hands a target to/// [`Driver::call_llm_target`]; the client rides along as@@ -285,37 +294,42 @@ pub trait Algorithm: Send + Sync + 'static {Ok(())}- /// Run one request as a stream of [`Step`]s (provided). The algorithm runs on its- /// own task; drive the stream: serve each [`Step::CallLlm`] (via its- /// [`default_client`](RoutedRequest::default_client) or your own transport) and read- /// [`Step::Decision`]s until the final [`Step::ReturnToAgent`]. The step channel is- /// bounded, so pulling paces the algorithm; each call is independent, so many run- /// concurrently.+ /// Process a request to completion, returning a stream of [`Step`]s.+ /// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve.+ /// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {- // This call's own driver: take its consumer stream, hand a producer-side clone to- // the algorithm task, and keep one to emit the terminal step. The task blocks- // publishing a step until the consumer pulls the previous one.let driver = Driver::new();- let stream = driver.stream();- tokio::spawn(async move {- let outcome = self- .create_run_task(ctx.clone(), driver.clone(), request)- .await;- let _ = driver.finish(ctx, outcome).await;- });- Box::pin(stream)- }-- /// Run one request to completion, serving each offloaded call with its- /// [`RoutedRequest::default_client`], and return the decision trace plus the final- /// [`Response`]. Provided: drives [`run_stream`](Self::run_stream) internally,- /// collecting each [`Step::Decision`]. Use it when the algorithm holds its own model- /// clients and the host wants the answer (and the decisions behind it); drive- /// [`run_stream`](Self::run_stream) instead to serve the calls yourself. The returned- /// [`Response`] is whatever the algorithm produced — a buffered [`LlmResponse::Agg`]- /// or a live [`LlmResponse::Stream`]; the caller decides whether to- /// [`into_agg`](LlmResponse::into_agg) it or forward its chunks. Errors if a routed- /// target has no client to serve its call, or the algorithm fails.+ let task_driver = driver.clone();+ let task_ctx = ctx.clone();+ let stream = task_driver.stream();+ let handle =+ tokio::spawn(async move { self.create_run_task(task_ctx, task_driver, request).await });+ // Dropping the stream aborts the algorithm task, so it doesn't keep running after the+ let abort_guard = AbortOnDrop(handle.abort_handle());++ let finish_driver = driver.clone();+ let finish_ctx = ctx.clone();+ let tail: StepStream = Box::pin(+ futures::stream::once(async move {+ let result = match handle.await {+ Ok(response) => response,+ Err(e) => Err(format!("Algorithm task panicked: {e}").into()),+ };+ finish_driver.finish(finish_ctx, result).await+ })+ .filter_map(|finish_result| async move { finish_result.err().map(Err) }),+ );++ let stream: StepStream = Box::pin(stream);+ Box::pin(futures::stream::select(stream, tail).map(move |step| {+ // link abort guard to stream+ let _keep_alive = &abort_guard;+ step+ }))+ }++ /// Process a request to completion, returning the final [`Response`] and the trace of+ /// [`Decision`]s the algorithm made along the way.async fn run(self: Arc<Self>,ctx: Context,@@ -788,6 +802,188 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn dropping_the_stream_cancels_the_algorithm_task(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ use std::sync::atomic::{AtomicBool, Ordering};+ use std::time::Duration;+ use tokio::sync::mpsc;++ // Sets a flag when dropped, so we can observe whether the algorithm task was+ // cancelled/dropped.+ struct DropGuard(Arc<AtomicBool>);+ impl Drop for DropGuard {+ fn drop(&mut self) {+ self.0.store(true, Ordering::SeqCst);+ }+ }++ struct StuckAlgo {+ started: mpsc::UnboundedSender<()>,+ dropped: Arc<AtomicBool>,+ }++ #[async_trait]+ impl Algorithm for StuckAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ _driver: Driver,+ _request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let _guard = DropGuard(self.dropped.clone());+ let _ = self.started.send(());+ // Await forever without ever touching the driver.+ std::future::pending::<()>().await;+ unreachable!()+ }+ }++ let (started_tx, mut started_rx) = mpsc::unbounded_channel();+ let dropped = Arc::new(AtomicBool::new(false));+ let algo: Arc<dyn Algorithm> = Arc::new(StuckAlgo {+ started: started_tx,+ dropped: dropped.clone(),+ });++ let stream = algo.run_stream(Context::default(), request());+ started_rx.recv().await.ok_or("task never started")?;+ drop(stream);+ tokio::time::sleep(Duration::from_millis(100)).await;++ assert!(+ dropped.load(Ordering::SeqCst),+ "algorithm task was NOT cancelled after dropping the stream"+ );+ Ok(())+ }++ #[tokio::test]+ async fn create_run_task_panic_surfaces_as_a_stream_error(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // An algorithm whose task panics must surface an `Err` step to the stream+ // consumer, not abort the process from an unobserved detached task.+ struct Panicky;++ #[async_trait]+ impl Algorithm for Panicky {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ _driver: Driver,+ _request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ panic!("boom");+ }+ }++ let algo: Arc<dyn Algorithm> = Arc::new(Panicky);+ let stream = algo.run_stream(Context::default(), request());+ tokio::pin!(stream);++ let mut saw_error = false;+ while let Some(step) = stream.next().await {+ match step {+ Err(err) => {+ assert!(err.to_string().contains("panicked"));+ saw_error = true;+ }+ Ok(_) => return Err("expected the panic to surface as an error step".into()),+ }+ }++ assert!(saw_error, "expected an error step from the panicked task");+ Ok(())+ }++ #[tokio::test]+ async fn run_returns_an_error_when_the_algorithm_task_panics(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // The panic surfaces as an `Err` step inside `run_stream`; `run` propagates it+ // via `?`, so the caller gets an `Err` rather than a hang or a silent panic.+ struct Panicky;++ #[async_trait]+ impl Algorithm for Panicky {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ _driver: Driver,+ _request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ panic!("boom");+ }+ }++ let algo: Arc<dyn Algorithm> = Arc::new(Panicky);+ match algo.run(Context::default(), request()).await {+ Ok(_) => Err("expected run to surface the algorithm panic as an error".into()),+ Err(err) => {+ assert!(err.to_string().contains("panicked"));+ Ok(())+ }+ }+ }++ #[tokio::test]+ async fn cancelling_run_cancels_the_algorithm_task() -> Result<(), Box<dyn Error + Send + Sync>>+ {+ use std::sync::atomic::{AtomicBool, Ordering};+ use std::time::Duration;+ use tokio::sync::mpsc;++ // Sets a flag when dropped, so we can observe whether the algorithm task was+ // cancelled once the `run` future driving it is dropped.+ struct DropGuard(Arc<AtomicBool>);+ impl Drop for DropGuard {+ fn drop(&mut self) {+ self.0.store(true, Ordering::SeqCst);+ }+ }++ struct StuckAlgo {+ started: mpsc::UnboundedSender<()>,+ dropped: Arc<AtomicBool>,+ }++ #[async_trait]+ impl Algorithm for StuckAlgo {+ async fn create_run_task(+ self: Arc<Self>,+ _ctx: Context,+ _driver: Driver,+ _request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let _guard = DropGuard(self.dropped.clone());+ let _ = self.started.send(());+ // Hang forever without ever touching the driver, so only cancellation+ // (not a dropped step channel) can stop this task.+ std::future::pending::<()>().await;+ unreachable!()+ }+ }++ let (started_tx, mut started_rx) = mpsc::unbounded_channel();+ let dropped = Arc::new(AtomicBool::new(false));+ let algo: Arc<dyn Algorithm> = Arc::new(StuckAlgo {+ started: started_tx,+ dropped: dropped.clone(),+ });++ // Drive `run` on its own task, wait until the algorithm task is up, then cancel+ // `run` — dropping its future (and the `run_stream` stream it holds).+ let run_task = tokio::spawn(async move { algo.run(Context::default(), request()).await });+ started_rx.recv().await.ok_or("task never started")?;+ run_task.abort();+ tokio::time::sleep(Duration::from_millis(100)).await;++ assert!(+ dropped.load(Ordering::SeqCst),+ "algorithm task was NOT cancelled after cancelling run"+ );+ Ok(())+ }+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]async fn run_caps_concurrent_calls() -> Result<(), Box<dyn Error + Send + Sync>> {use std::sync::atomic::{AtomicUsize, Ordering};