Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): run should return when stream ends (#124)
Greg Clark880f9beaa05a
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex b8eccc0c4..d3f42765f 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -344,10 +344,6 @@ wherectx: Context<S>,request: Request,) -> Result<(Vec<Arc<dyn Decision>>, Response), Box<dyn Error + Send + Sync>> {- // Serve up to this many offloaded calls concurrently, so an algorithm that- // fans out (e.g. an ensemble) isn't serialized on the client.- const MAX_CONCURRENT_CALLS: usize = 10;-// Serve one offloaded call with its target's default client. A failed *model*// call is forwarded to the algorithm via `respond`; this errors only on an// infrastructure failure (no default client, or the promise was dropped).@@ -372,31 +368,28 @@ wherelet mut trace: Vec<Arc<dyn Decision>> = Vec::new();let mut in_flight = futures::stream::FuturesUnordered::new();let mut final_response: Option<Response> = None;- let mut stream_open = true;- while stream_open || !in_flight.is_empty() {+ loop {tokio::select! {- // Surface a failed serve as soon as it completes.- Some(result) = in_flight.next(), if !in_flight.is_empty() => result?,- // Pull the next step, unless the stream ended or we're at the cap.- step = stream.next(), if stream_open && in_flight.len() < MAX_CONCURRENT_CALLS => {+ Some(result) = in_flight.next() => match result {+ Ok(()) => {}, // CallLlm completed successfully+ Err(err) => return Err(err), // CallLlm failed, propagate the error+ },+ step = stream.next() => {match step {- None => stream_open = false,+ None => break, // stream has ended, no more stepsSome(item) => match item? {Step::CallLlm(call) => in_flight.push(serve(*call)),Step::Decision(decision) => trace.push(decision),Step::ReturnToAgent(response) => {final_response = Some(*response);- stream_open = false;+ break;}- },+ }}- }+ },}}-- // Return whatever the algorithm produced — buffered or streamed. The caller- // decides whether to aggregate the response or forward its chunks.final_response.map(|response| (trace, response)).ok_or_else(|| "run: stream ended without a final response".into())@@ -993,144 +986,246 @@ mod tests {Ok(())}- #[tokio::test(flavor = "multi_thread", worker_threads = 4)]- async fn run_caps_concurrent_calls() -> Result<(), Box<dyn Error + Send + Sync>> {+ // --- first-wins hedging: `run` must not wait on losing speculative calls -------------++ /// The loser: signals it has started serving (so the winner can then win with the+ /// loser's serve guaranteed in flight), then finishes late (`Some(delay)`) or never+ /// (`None`).+ struct LoserClient {+ started: Arc<tokio::sync::Notify>,+ delay: Option<std::time::Duration>,+ }++ #[async_trait]+ impl RoutedLlmClient for LoserClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ self.started.notify_one();+ match self.delay {+ Some(delay) => tokio::time::sleep(delay).await,+ None => std::future::pending::<()>().await,+ }+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ decision.selected_model().to_string(),+ )),+ metadata: None,+ })+ }+ }++ /// The winner: waits until the loser's serve has started, then echoes immediately, so+ /// the loser's serve is guaranteed in flight when the winner wins.+ struct GatedEchoClient {+ gate: Arc<tokio::sync::Notify>,+ }++ #[async_trait]+ impl RoutedLlmClient for GatedEchoClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ self.gate.notified().await;+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ decision.selected_model().to_string(),+ )),+ metadata: None,+ })+ }+ }++ /// Offloads two targets concurrently and returns the first to resolve, dropping the+ /// loser's call (first-wins hedging).+ struct Hedge {+ winner: LlmTarget,+ loser: LlmTarget,+ }++ #[async_trait]+ impl Algorithm for Hedge {+ async fn create_run_task(+ self: Arc<Self>,+ ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response, Box<dyn Error + Send + Sync>> {+ let dec_w: Arc<dyn Decision> = Arc::new(TestDecision {+ model: self.winner.semantic_name.clone(),+ });+ let dec_l: Arc<dyn Decision> = Arc::new(TestDecision {+ model: self.loser.semantic_name.clone(),+ });+ let win = driver.call_llm_target(ctx.clone(), &self.winner, request.clone(), dec_w);+ let lose = driver.call_llm_target(ctx, &self.loser, request, dec_l);+ // First to resolve wins; `select!` drops the losing future (and its promise).+ tokio::select! {+ res = win => res,+ res = lose => res,+ }+ }+ }++ /// Builds a hedging algo whose winner is gated behind the loser starting, and whose+ /// loser finishes after `loser_delay` (or never, when `None`).+ fn hedge(loser_delay: Option<std::time::Duration>) -> Arc<dyn Algorithm> {+ let started = Arc::new(tokio::sync::Notify::new());+ let winner = LlmTarget {+ semantic_name: "winner".to_string(),+ llm_client: Some(Arc::new(GatedEchoClient {+ gate: started.clone(),+ })),+ };+ let loser = LlmTarget {+ semantic_name: "loser".to_string(),+ llm_client: Some(Arc::new(LoserClient {+ started,+ delay: loser_delay,+ })),+ };+ Arc::new(Hedge { winner, loser })+ }++ #[tokio::test]+ async fn run_returns_the_winner_without_a_late_loser_overwriting_it(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // The loser responds 50ms after the winner has already won. `run` must return the+ // winner, not the loser's `respond`-to-a-dropped-receiver error.+ let (_trace, response) = hedge(Some(std::time::Duration::from_millis(50)))+ .run(Context::default(), request())+ .await?;+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "winner"+ );+ Ok(())+ }++ #[tokio::test]+ async fn run_returns_the_winner_without_hanging_on_a_pending_loser(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {+ // The loser never resolves. `run` must return the winner promptly, not hang+ // waiting for the in-flight loser.+ let run = hedge(None).run(Context::default(), request());+ let (_trace, response) = tokio::time::timeout(std::time::Duration::from_secs(1), run)+ .await+ .map_err(|_| "run hung waiting for a pending loser")??;+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "winner"+ );+ Ok(())+ }++ #[tokio::test]+ async fn run_surfaces_a_terminal_error_with_many_calls_in_flight(+ ) -> Result<(), Box<dyn Error + Send + Sync>> {use std::sync::atomic::{AtomicUsize, Ordering};- use std::time::Duration;- use tokio::sync::{mpsc, Semaphore};-- // Mirrors `run`'s private `MAX_CONCURRENT_CALLS`. The algorithm fans out more- // calls than the cap; only `CAP` should ever be in flight, the rest wait.- const CAP: usize = 10;- const TOTAL_CALLS: usize = CAP + 5;-- // Records peak concurrency and holds each call open until the gate is released,- // so calls pile up and the cap can be observed.- struct ProbeClient {- current: Arc<AtomicUsize>,- max: Arc<AtomicUsize>,- entered: mpsc::UnboundedSender<()>,- gate: Arc<Semaphore>,++ // A large fan-out (10 matched the old, now-removed concurrency cap). The terminal+ // error must still reach the caller with all of these calls pending.+ const N: usize = 10;++ // Enters each call; once all N are in flight, signals, then pends forever.+ struct EnterThenPend {+ started: Arc<AtomicUsize>,+ all_started: Arc<tokio::sync::Notify>,+ n: usize,}#[async_trait]- impl RoutedLlmClient for ProbeClient {+ impl RoutedLlmClient for EnterThenPend {async fn call(&self,_ctx: Context,_request: Request,- decision: Arc<dyn Decision>,+ _decision: Arc<dyn Decision>,) -> Result<Response, Box<dyn Error + Send + Sync>> {- let now = self.current.fetch_add(1, Ordering::SeqCst) + 1;- self.max.fetch_max(now, Ordering::SeqCst);- let _ = self.entered.send(());- // Block until the test opens the gate.- let permit = self.gate.acquire().await.map_err(|e| e.to_string())?;- drop(permit);- self.current.fetch_sub(1, Ordering::SeqCst);- Ok(Response {- llm_response: LlmResponse::Agg(text_response(- None,- decision.selected_model().to_string(),- )),- metadata: None,- })+ if self.started.fetch_add(1, Ordering::SeqCst) + 1 == self.n {+ self.all_started.notify_one();+ }+ std::future::pending::<()>().await;+ unreachable!()}}- // Fans out `n` concurrent calls to one target.- struct FanOut {- n: usize,+ // Fans out N calls, then errors as soon as all N are in flight — exercising a+ // terminal failure emitted while the offloaded calls are still pending.+ struct FanOutThenError {target: LlmTarget,+ all_started: Arc<tokio::sync::Notify>,+ n: usize,}#[async_trait]- impl Algorithm for FanOut {+ impl Algorithm for FanOutThenError {async fn create_run_task(self: Arc<Self>,ctx: Context,driver: Driver,- _request: Request,+ request: Request,) -> Result<Response, Box<dyn Error + Send + Sync>> {- let calls = (0..self.n).map(|i| {- let driver = driver.clone();- let ctx = ctx.clone();- let target = self.target.clone();- async move {- let decision: Arc<dyn Decision> = Arc::new(TestDecision {- model: format!("m{i}"),- });- driver- .call_llm_target(ctx, &target, request(), decision)- .await+ let offloads = futures::future::join_all((0..self.n).map(|i| {+ let decision: Arc<dyn Decision> = Arc::new(TestDecision {+ model: format!("m{i}"),+ });+ driver.call_llm_target(ctx.clone(), &self.target, request.clone(), decision)+ }));+ tokio::select! {+ _ = offloads => Err("offloads unexpectedly completed".into()),+ _ = self.all_started.notified() => {+ Err("terminal error while calls pending".into())}- });- futures::future::join_all(calls).await;- Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, "done".to_string())),- metadata: None,- })+ }}}- let current = Arc::new(AtomicUsize::new(0));- let max = Arc::new(AtomicUsize::new(0));- let (entered_tx, mut entered_rx) = mpsc::unbounded_channel();- let gate = Arc::new(Semaphore::new(0)); // starts closed-- let client = Arc::new(ProbeClient {- current: current.clone(),- max: max.clone(),- entered: entered_tx,- gate: gate.clone(),- }) as Arc<dyn RoutedLlmClient>;+ let all_started = Arc::new(tokio::sync::Notify::new());let target = LlmTarget {- semantic_name: "m".to_string(),- llm_client: Some(client),+ semantic_name: "pending".to_string(),+ llm_client: Some(Arc::new(EnterThenPend {+ started: Arc::new(AtomicUsize::new(0)),+ all_started: all_started.clone(),+ n: N,+ })),};-- let algo: Arc<dyn Algorithm> = Arc::new(FanOut {- n: TOTAL_CALLS,+ let algo: Arc<dyn Algorithm> = Arc::new(FanOutThenError {target,+ all_started,+ n: N,});- let handle = tokio::spawn(async move { algo.run(Context::default(), request()).await });- // Exactly `CAP` calls should enter; each `recv` times out into an error if the- // run dispatched fewer than the cap.- for _ in 0..CAP {- tokio::time::timeout(Duration::from_secs(5), entered_rx.recv())- .await- .map_err(|_| "timed out waiting for a call to enter")?;+ // With the cap gone, `run` keeps polling the stream even with N calls in flight, so+ // the terminal error surfaces promptly instead of hanging.+ let run = algo.run(Context::default(), request());+ let result = tokio::time::timeout(std::time::Duration::from_millis(500), run)+ .await+ .map_err(|_| "run hung: terminal error not surfaced with the cap full")?;+ match result {+ Ok(_) => Err("expected the terminal error, got a response".into()),+ Err(err) => {+ assert!(err+ .to_string()+ .contains("terminal error while calls pending"));+ Ok(())+ }}- assert_eq!(- current.load(Ordering::SeqCst),- CAP,- "exactly the cap should be in flight"- );- // No further call enters while the cap is saturated — extra steps wait for capacity.- assert!(- tokio::time::timeout(Duration::from_millis(200), entered_rx.recv())- .await- .is_err(),- "a call beyond the cap entered while {CAP} were already in flight"- );-- // Release the gate; every call completes and the run finishes.- gate.add_permits(TOTAL_CALLS);- let (_trace, response) = tokio::time::timeout(Duration::from_secs(5), handle).await???;- assert_eq!(- response- .llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default(),- "done"- );- assert_eq!(- max.load(Ordering::SeqCst),- CAP,- "peak in-flight should equal the cap"- );- Ok(())}}