Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore: Simplify `run_stream`, rename `Driver::info` to `Driver::decide`. (#364)

Graham King58f355a132d6
diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs
index 58270cc03..e20d1a885 100644
--- a/crates/libsy-llm-client/src/observability.rs
+++ b/crates/libsy-llm-client/src/observability.rs
@@ -176,7 +176,6 @@ fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {
LibsyError::NoTargets => Cow::Borrowed("no_targets"),
LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),
LibsyError::Driver(_) => Cow::Borrowed("driver_error"),
- LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),
LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),
LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),
LibsyError::External { .. } => Cow::Borrowed("_OTHER"),
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 8e3c66970..daa84dbfa 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -454,7 +454,7 @@ impl Algorithm for SingleCallAlgo {
Some(format!("picked '{}'", target.semantic_name)),
true,
);
- driver.info(decision.clone()).await?;
+ driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
}
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index f1012fb04..ec9d48fb2 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -349,7 +349,7 @@ where
)),
true,
);
- driver.info(decision.clone()).await?;
+ driver.decide(decision.clone()).await?;
// 4. Post-decision replay: every processor sees the decision so stateful ones
// can bind it, and may rewrite the outbound request (e.g. add a target prompt).
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index a14176b28..1ffc32484 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -32,7 +32,7 @@ impl Algorithm for Noop {
Some("noop returned its synthetic response".to_string()),
true,
);
- driver.info(decision.clone()).await?;
+ driver.decide(decision.clone()).await?;
let llm_response = LlmResponse::Agg(AggLlmResponse {
id: Some("switchyard-noop".to_string()),
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 1c7e322b1..d80b16f17 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -38,7 +38,7 @@ impl Algorithm for Passthrough {
)),
true,
);
- driver.info(decision.clone()).await?;
+ driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
}
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 519b1e4bb..f63d6e107 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -8,13 +8,14 @@
use std::{
collections::{HashMap, HashSet},
future::Future,
+ panic::AssertUnwindSafe,
pin::Pin,
sync::Arc,
time::Instant,
};
use async_trait::async_trait;
-use futures::{Stream, StreamExt};
+use futures::{FutureExt, Stream, StreamExt};
use parking_lot::Mutex;
use tokio::sync::{mpsc, oneshot};
use tokio_stream::wrappers::ReceiverStream;
@@ -184,7 +185,7 @@ impl Driver {
/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
/// Each successfully published decision is counted and logged with its
/// reasoning; a decision the stream never accepted is not recorded.
- pub async fn info(&self, decision: Decision) -> Result<()> {
+ pub async fn decide(&self, decision: Decision) -> Result<()> {
self.step_tx
.send(Ok(Step::Decision(decision.clone())))
.await
@@ -210,7 +211,7 @@ pub enum Step {
/// The algorithm needs this model call performed. The host serves it and fulfills
/// it with [`CallModel::respond`]. Boxed: it is by far the largest variant.
CallModel(Box<CallModel>),
- /// A routing decision the algorithm made, published via [`Driver::info`] as it
+ /// A routing decision the algorithm made, published via [`Driver::decide`] as it
/// happens (rather than collected into a trace returned at the end).
Decision(Decision),
/// The algorithm finished with its final response — the last step of a run.
@@ -271,6 +272,15 @@ where
.ok_or(LibsyError::MissingFinalResponse)
}
+/// Recover the message from an algorithm's panic.
+fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
+ payload
+ .downcast_ref::<&'static str>()
+ .map(|message| (*message).to_string())
+ .or_else(|| payload.downcast_ref::<String>().cloned())
+ .unwrap_or_else(|| "unknown panic payload".to_string())
+}
+
/// Abort guard
struct AbortOnDrop(tokio::task::AbortHandle);
@@ -513,7 +523,7 @@ pub(crate) async fn call_model_with_fallback(
};
decision = fallback_decision(&target, &next, reason);
target = next;
- driver.info(decision.clone()).await?;
+ driver.decide(decision.clone()).await?;
}
}
@@ -546,50 +556,45 @@ pub trait Algorithm: Send + Sync + 'static {
fn name(&self) -> &str;
/// Run one request to completion: make model calls with [`Driver::call_model`],
- /// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
+ /// publish [`Decision`]s with [`Driver::decide`], and return the final [`Response`].
/// The method an algorithm implements; [`run_stream`](Self::run_stream) drives it.
async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response>;
/// Process a request to completion, returning a stream of [`Step`]s.
///
/// The consumer must fulfill every [`Step::CallModel`] before the algorithm can
- /// continue. The bounded step channel applies backpressure when the consumer is
- /// not polling. A successful run ends with [`Step::Done`]; a failure is
- /// emitted as an `Err` item. Dropping the stream aborts the spawned algorithm task.
+ /// continue. Every run ends with exactly one terminal item — [`Step::Done`] on
+ /// success, an `Err` item on failure, including when the algorithm panics. Dropping
+ /// the stream aborts the spawned algorithm task.
///
/// Every invocation owns a separate [`Driver`].
fn run_stream(self: Arc<Self>, request: Request) -> StepStream {
let (driver, step_rx) = Driver::new(self.name());
- let task_driver = driver.clone();
- let stream = ReceiverStream::new(step_rx);
- // One `libsy.run` span covers the whole algorithm task; the driver's
- // `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s
- // contextual parenting.
let span = observability::run_span(self.name(), &request);
let handle = tokio::spawn(
async move {
let algorithm = self.name().to_string();
- observability::observe_run(&algorithm, self.route(task_driver, request)).await
+ // Catch a panicking algorithm so the run still publishes a terminal step.
+ let route = AssertUnwindSafe(self.route(driver.clone(), request)).catch_unwind();
+ let result = observability::observe_run(&algorithm, async move {
+ route.await.unwrap_or_else(|payload| {
+ Err(LibsyError::AlgorithmError {
+ message: format!(
+ "algorithm task panicked: {}",
+ panic_message(payload.as_ref())
+ ),
+ })
+ })
+ })
+ .await;
+
+ let _ = driver.finish(result).await;
}
.instrument(span),
);
// Dropping the stream aborts the algorithm task when its consumer goes away.
let abort_guard = AbortOnDrop(handle.abort_handle());
-
- let finish_driver = driver.clone();
- let tail: StepStream = Box::pin(
- futures::stream::once(async move {
- let result = match handle.await {
- Ok(response) => response,
- Err(source) => Err(LibsyError::AlgorithmTask { source }),
- };
- finish_driver.finish(result).await
- })
- .filter_map(|finish_result| async move { finish_result.err().map(Err) }),
- );
-
- let stream: StepStream = Box::pin(stream);
- Box::pin(futures::stream::select(stream, tail).map(move |step| {
+ Box::pin(ReceiverStream::new(step_rx).map(move |step| {
// link abort guard to stream
let _keep_alive = &abort_guard;
step
@@ -694,7 +699,7 @@ mod tests {
.ok_or(LibsyError::NoTargets)?
.clone();
let decision = test_decision(target.semantic_name.clone());
- driver.info(decision.clone()).await?;
+ driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
}
@@ -765,10 +770,10 @@ mod tests {
let first_response = first
.await
- .map_err(|source| LibsyError::AlgorithmTask { source })??;
+ .map_err(|source| LibsyError::external("joining a test task", source))??;
let second_response = second
.await
- .map_err(|source| LibsyError::AlgorithmTask { source })??;
+ .map_err(|source| LibsyError::external("joining a test task", source))??;
assert_eq!(
first_response.llm_response.as_agg().map(completion_text),
Some("first response".to_string())
@@ -792,7 +797,7 @@ mod tests {
drop(call);
let result = producer
.await
- .map_err(|source| LibsyError::AlgorithmTask { source })?;
+ .map_err(|source| LibsyError::external("joining a test task", source))?;
assert!(matches!(
result,
Err(LibsyError::Driver(DriverError::ResponseDropped))
@@ -802,7 +807,7 @@ mod tests {
let (driver, step_rx) = Driver::new("test");
drop(step_rx);
let decision = test_decision("closed".to_string());
- let result = driver.info(decision).await;
+ let result = driver.decide(decision).await;
assert!(matches!(
result,
Err(LibsyError::Driver(DriverError::StreamClosed))
@@ -837,7 +842,7 @@ mod tests {
let result = producer
.await
- .map_err(|source| LibsyError::AlgorithmTask { source })?;
+ .map_err(|source| LibsyError::external("joining a test task", source))?;
assert!(matches!(
result,
Err(LibsyError::Driver(DriverError::Abandoned))
@@ -1049,7 +1054,7 @@ mod tests {
let completion = tokio::time::timeout(Duration::from_secs(5), handle)
.await
.map_err(|error| LibsyError::external("waiting for test task", error))?
- .map_err(|source| LibsyError::AlgorithmTask { source })??;
+ .map_err(|source| LibsyError::external("joining a test task", source))??;
assert_eq!(completion, "m");
}
Ok(())
@@ -1150,8 +1155,8 @@ mod tests {
#[tokio::test]
async fn route_panic_surfaces_as_a_stream_error() -> Result<()> {
- // An algorithm whose task panics must surface an `Err` step to the stream
- // consumer, not abort the process from an unobserved detached task.
+ // An algorithm whose task panics must surface an `Err` step carrying the panic
+ // message, not abort the process from an unobserved detached task.
struct Panicky;
#[async_trait]
@@ -1177,7 +1182,8 @@ mod tests {
while let Some(step) = stream.next().await {
match step {
Err(err) => {
- assert!(matches!(err, LibsyError::AlgorithmTask { .. }));
+ // The panic message is preserved, not flattened into an opaque failure.
+ assert!(err.to_string().contains("algorithm task panicked: boom"));
saw_error = true;
}
Ok(_) => return Err(test_error("expected the panic to surface as an error step")),
@@ -1188,34 +1194,45 @@ mod tests {
Ok(())
}
+ /// A panicking algorithm must publish its terminal step even when it left a `Driver`
+ /// clone alive in another task. That clone holds the step channel open, so a run that
+ /// merely unwound would never terminate and the consumer would wait forever.
#[tokio::test]
- async fn run_returns_an_error_when_the_algorithm_task_panics() -> Result<()> {
- // The panic surfaces as an `Err` step inside `run_stream`; `run` propagates it
- // via `?`, so the caller gets an `Err` rather than a hang or a silent panic.
- struct Panicky;
+ async fn a_panic_with_a_leaked_driver_clone_still_terminates_the_run() -> Result<()> {
+ struct LeakyPanic;
#[async_trait]
- impl Algorithm for Panicky {
+ impl Algorithm for LeakyPanic {
fn name(&self) -> &str {
- "panicky"
+ "leaky_panic"
}
- async fn route(
- self: Arc<Self>,
- _driver: Driver,
- _request: Request,
- ) -> Result<Response> {
+ async fn route(self: Arc<Self>, driver: Driver, _request: Request) -> Result<Response> {
+ tokio::spawn(async move {
+ // Outlives the panic below, keeping a sender clone alive.
+ let _keep_alive = driver;
+ std::future::pending::<()>().await;
+ });
+ tokio::task::yield_now().await;
panic!("boom");
}
}
- let algo: Arc<dyn Algorithm> = Arc::new(Panicky);
- match test_drive(algo, request(), echo()).await {
+ let algo: Arc<dyn Algorithm> = Arc::new(LeakyPanic);
+ // The timeout turns the hang this guards against into a failure rather than a hang.
+ let result = tokio::time::timeout(
+ std::time::Duration::from_secs(1),
+ test_drive(algo, request(), echo()),
+ )
+ .await
+ .map_err(|error| LibsyError::external("waiting for the panicked run to end", error))?;
+
+ match result {
Ok(_) => Err(test_error(
- "expected the run to surface the algorithm panic as an error",
+ "expected the panic to end the run with an error",
)),
Err(err) => {
- assert!(matches!(err, LibsyError::AlgorithmTask { .. }));
+ assert!(err.to_string().contains("algorithm task panicked: boom"));
Ok(())
}
}
diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs
index e77dfe80c..4703cb445 100644
--- a/crates/libsy/src/error.rs
+++ b/crates/libsy/src/error.rs
@@ -36,14 +36,6 @@ pub enum LibsyError {
#[error(transparent)]
Driver(#[from] DriverError),
- /// The spawned algorithm task failed before returning normally.
- #[error("algorithm task failed: {source}")]
- AlgorithmTask {
- /// Tokio task failure, including panic and unexpected cancellation details.
- #[from]
- source: tokio::task::JoinError,
- },
-
/// An algorithm's step stream ended without a terminal response.
#[error("algorithm run ended without a final response")]
MissingFinalResponse,
@@ -95,7 +87,7 @@ impl LibsyError {
}
/// Failures in the step-stream driver.
-#[derive(Debug, Error, PartialEq, Eq)]
+#[derive(Debug, Error)]
pub enum DriverError {
/// The consumer side of the step channel was dropped.
#[error("driver stream is closed")]