Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(examples): forward full ensemble requests and cap concurrent explorations (#202)
Elyas Mehtabuddin8d47c2f3e52a
diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rsindex 27520403f..7f96b1d8f 100644--- a/crates/libsy/examples/ensemble.rs+++ b/crates/libsy/examples/ensemble.rs@@ -13,10 +13,11 @@//! NVIDIA_API_KEY=... cargo run -p switchyard-libsy --example ensemble//!//! Unlike the reference routers, this algorithm is **stateful**: the win tally,-//! turn counter, and committed choice live behind a [`parking_lot::Mutex`] so one-//! shared `&self` can serve a session's requests concurrently (see the-//! `Algorithm` docs). In a proxy setup one `EnsembleOrchAlgo` is created per session,-//! so this state is per-session.+//! the claimed exploration count, and the committed choice live behind a+//! [`parking_lot::Mutex`] so one shared `&self` can serve a session's+//! requests concurrently (see the `Algorithm` docs); a [`tokio::sync::Notify`]+//! wakes a request waiting for the exploration budget to finish. In a proxy setup+//! one `EnsembleOrchAlgo` is created per session, so this state is per-session.use std::collections::BTreeMap;use std::sync::Arc;@@ -24,11 +25,13 @@ use std::sync::Arc;use async_trait::async_trait;use parking_lot::Mutex;+use tokio::sync::Notify;+use switchyard_libsy::{Algorithm, Driver, LibsyError, LlmTarget, LlmTargetSet, Result};use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};use switchyard_protocol::{- Context, Decision, Request, Response, RoutedLlmClient, completion_text, prompt_text,- text_request,+ Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,+ prompt_text, text_request,};const CANDIDATE_MODELS: [&str; 3] = [@@ -95,12 +98,44 @@ impl Decision for EnsembleDecision {struct EnsembleState {/// Judge-win count per candidate model.wins: BTreeMap<String, u64>,- /// Completed ensemble (exploration) turns.- turns: u64,+ /// Running or completed exploration turns. Failed turns return their slot.+ used_slots: u64,/// The model committed to once exploration is over; `None` while exploring.committed: Option<String>,}+/// How one request is served: straight to the committed model, or as a fresh+/// exploration turn (a slot was reserved for it under the lock).+enum RoutePlan {+ Committed(String),+ Explore,+}++/// Holds a reserved exploration slot for the length of one turn and returns it if+/// the turn is dropped before it tallies a winner — the run future aborted on a+/// client disconnect, a panic, or an error. A turn that completes calls+/// [`disarm`](ReservationGuard::disarm) and keeps its now-counted slot.+struct ReservationGuard<'a> {+ algo: &'a EnsembleOrchAlgo,+ armed: bool,+}++impl ReservationGuard<'_> {+ /// Give up the claim on the slot: the turn completed and counted it, so it+ /// must not be released.+ fn disarm(&mut self) {+ self.armed = false;+ }+}++impl Drop for ReservationGuard<'_> {+ fn drop(&mut self) {+ if self.armed {+ self.algo.release_reservation();+ }+ }+}+/// Ensemble router: fan out to candidates, judge the best, then commit.pub struct EnsembleOrchAlgo {candidate_models: Vec<String>,@@ -110,6 +145,9 @@ pub struct EnsembleOrchAlgo {exploration_turns: u64,target_set: LlmTargetSet,state: Mutex<EnsembleState>,+ /// Notified whenever a reserved exploration finishes, so a request waiting+ /// for the exploration budget to drain can recheck and commit.+ explorations_done: Notify,}impl EnsembleOrchAlgo {@@ -132,30 +170,59 @@ impl EnsembleOrchAlgo {target_set,state: Mutex::new(EnsembleState {wins: BTreeMap::new(),- turns: 0,+ used_slots: 0,committed: None,}),+ explorations_done: Notify::new(),}}- /// If exploration is over, return the committed model (committing lazily on- /// the first post-exploration request). `None` means keep ensembling.+ /// Decide how to serve one request, reserving an exploration slot under the+ /// lock before any async work begins. This caps concurrent explorations at+ /// `exploration_turns` and commits only from a complete tally:///- /// The lock is taken and dropped here, never across an `await`. Under- /// concurrency two requests may both commit; they pick the same model from- /// the same tally (with a stable tie-break), so the result is identical.- fn resolve_committed(&self) -> Result<Option<String>> {- let mut state = self.state.lock();- if let Some(model) = &state.committed {- return Ok(Some(model.clone()));- }- // `exploration_turns == 0` keeps the algorithm in ensemble mode forever.- if self.exploration_turns > 0 && state.turns >= self.exploration_turns {- let best = self.pick_best(&state.wins)?;- state.committed = Some(best.clone());- return Ok(Some(best));+ /// - already committed → route to that model;+ /// - a slot is free (`used_slots < exploration_turns`, or unlimited when+ /// `exploration_turns == 0`) → reserve it and explore;+ /// - the budget is fully reserved but some explorations are still running →+ /// wait for them rather than commit early or start an extra turn;+ /// - the budget is fully reserved and every reservation has been tallied →+ /// commit to the winningest model.+ ///+ /// The lock is only held for the synchronous decision; the wait happens after+ /// it is released, with the wakeup registered first so no completion is missed.+ async fn plan_route(&self) -> Result<RoutePlan> {+ loop {+ let wait = self.explorations_done.notified();+ tokio::pin!(wait);+ {+ let mut state = self.state.lock();+ if let Some(model) = &state.committed {+ return Ok(RoutePlan::Committed(model.clone()));+ }+ // `exploration_turns == 0` keeps the algorithm ensembling forever.+ if self.exploration_turns == 0 || state.used_slots < self.exploration_turns {+ state.used_slots += 1;+ return Ok(RoutePlan::Explore);+ }+ if state.wins.values().sum::<u64>() >= self.exploration_turns {+ let best = self.pick_best(&state.wins)?;+ state.committed = Some(best.clone());+ return Ok(RoutePlan::Committed(best));+ }+ // Register for the next completion before dropping the lock so a+ // notify between here and the await below is not lost.+ wait.as_mut().enable();+ }+ wait.await;}- Ok(None)+ }++ /// Return an exploration slot whose turn failed without tallying a winner, so+ /// a waiting request can take it and the budget is not spent by a failed turn.+ fn release_reservation(&self) {+ self.state.lock().used_slots -= 1;+ self.explorations_done.notify_waiters();}/// The candidate with the most judge-wins, breaking ties toward the earlier@@ -191,39 +258,23 @@ impl EnsembleOrchAlgo {selected_model: model.clone(),phase: EnsemblePhase::Committed,});- // The agent's inbound name rides through; the committed model is on the- // decision, not stamped onto the request.- let routed = Request {- llm_request: text_request(- request.llm_request.model.clone(),- prompt_text(&request.llm_request),- ),- raw_request: request.raw_request,- metadata: request.metadata,- };+ // Forward the caller's complete request unchanged; the committed model is+ // carried on the decision, not stamped onto the request.let response = driver- .call_llm_target(ctx, &target, routed, decision.clone())+ .call_llm_target(ctx, &target, request, decision.clone()).await?;Ok((vec![decision], response))}- /// One exploration turn: fan out to every candidate, judge the survivors,- /// tally the winner, and return its response.- async fn ensemble_turn(+ /// Call all candidates concurrently and buffer successful responses. A call+ /// or stream error removes only that candidate from the turn.+ async fn fan_out_candidates(&self,driver: &Driver,- ctx: Context,- request: Request,- ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {- let user_prompt = prompt_text(&request.llm_request);- // The agent's inbound name rides through every sub-call unchanged; the model- // each call hits is carried by its decision, not stamped onto the request.- let inbound = request.llm_request.model.clone();-- // Fan out to all candidates concurrently with the same user prompt. Each- // call is annotated with its own candidate decision so the caller can see- // which model an offloaded call targets.- let mut candidate_decisions: Vec<Arc<dyn Decision>> = Vec::new();+ ctx: &Context,+ request: &Request,+ ) -> Result<(Vec<Arc<dyn Decision>>, Vec<(String, Response)>)> {+ let mut decisions: Vec<Arc<dyn Decision>> = Vec::new();let mut calls = Vec::new();for model in &self.candidate_models {let target = self.target_set.get_target(model)?;@@ -232,33 +283,53 @@ impl EnsembleOrchAlgo {reasoning: format!("ensemble candidate '{model}'"),phase: EnsemblePhase::Candidate,});- candidate_decisions.push(decision.clone());- let call_request = Request {- llm_request: text_request(inbound.clone(), user_prompt.clone()),- raw_request: request.raw_request.clone(),- metadata: request.metadata.clone(),- };+ decisions.push(decision.clone());+ let request = request.clone();let model = model.clone();let ctx = ctx.clone();calls.push(async move {- (+ // Drain each stream inside its candidate future so slow streams do+ // not block the other candidates from making progress.+ let Response {+ llm_response,+ metadata,+ } = driver+ .call_llm_target(ctx, &target, request, decision)+ .await+ .ok()?;+ let llm_response = LlmResponse::Agg(llm_response.into_agg().await.ok()?);+ Some((model,- driver- .call_llm_target(ctx, &target, call_request, decision)- .await,- )+ Response {+ llm_response,+ metadata,+ },+ ))});}- let results = futures::future::join_all(calls).await;-- // Keep only successful responses, preserving candidate order. A failed- // candidate is simply excluded from judging rather than failing the turn.- let mut survivors: Vec<(String, Response)> = Vec::new();- for (model, result) in results {- if let Ok(response) = result {- survivors.push((model, response));- }- }++ let survivors = futures::future::join_all(calls)+ .await+ .into_iter()+ .flatten()+ .collect();+ Ok((decisions, survivors))+ }++ /// One exploration turn: fan out to every candidate, judge the survivors,+ /// tally the winner, and return its response.+ async fn ensemble_turn(+ &self,+ driver: &Driver,+ ctx: Context,+ request: Request,+ ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {+ let user_prompt = prompt_text(&request.llm_request);+ // Keep the caller's model ID; each decision selects the actual target.+ let inbound = request.llm_request.model.clone();++ let (candidate_decisions, survivors) =+ self.fan_out_candidates(driver, &ctx, &request).await?;if survivors.is_empty() {return Err(ensemble_error("all candidate calls failed"));}@@ -279,7 +350,7 @@ impl EnsembleOrchAlgo {phase: EnsemblePhase::Judge,});let judge_request = Request {- llm_request: text_request(inbound.clone(), judge_prompt),+ llm_request: text_request(inbound, judge_prompt),raw_request: request.raw_request.clone(),metadata: request.metadata.clone(),};@@ -302,13 +373,13 @@ impl EnsembleOrchAlgo {(model, response, Some(judge_decision))};- // Record the win and advance the turn counter under the lock (not held- // across any await).- {- let mut state = self.state.lock();- *state.wins.entry(winner_model.clone()).or_insert(0) += 1;- state.turns += 1;- }+ *self+ .state+ .lock()+ .wins+ .entry(winner_model.clone())+ .or_insert(0) += 1;+ self.explorations_done.notify_waiters();let winner_decision: Arc<dyn Decision> = Arc::new(EnsembleDecision {reasoning: format!("judge selected '{winner_model}' as best response"),@@ -383,17 +454,30 @@ impl Algorithm for EnsembleOrchAlgo {driver: Driver,request: Request,) -> Result<Response> {- // Fast path: exploration is over — route straight to the committed model;- // otherwise run a full ensemble turn. Both return a decision trace plus the- // final response.- let (trace, response) = if let Some(model) = self.resolve_committed()? {- self.route_committed(&driver, ctx.clone(), request, model)- .await?- } else {- self.ensemble_turn(&driver, ctx.clone(), request).await?+ // Reserve the route (committed model or a fresh exploration slot) before any+ // async work, then serve it. A reserved exploration that fails releases its+ // slot so the budget is not spent by a failed turn.+ let (trace, response) = match self.plan_route().await? {+ RoutePlan::Committed(model) => {+ self.route_committed(&driver, ctx.clone(), request, model)+ .await?+ }+ RoutePlan::Explore => {+ // Hold the reserved slot across the turn: an abort or panic+ // mid-turn returns it through the guard's `Drop`, and an error+ // returns it on the way out. A turn that completes disarms the+ // guard and keeps its now-counted slot.+ let mut guard = ReservationGuard {+ algo: &self,+ armed: true,+ };+ let result = self.ensemble_turn(&driver, ctx.clone(), request).await?;+ guard.disarm();+ result+ }};// Publish the trace to the stream (candidate..., judge?, winner). The- // candidate decisions also rode along on their offloaded `CallLlm` steps.+ // candidate decisions were also attached to their `CallLlm` steps.for decision in trace {driver.info(ctx.clone(), decision).await?;}@@ -467,11 +551,16 @@ async fn main() -> Result<()> {#[cfg(test)]mod tests {use super::*;+ use std::sync::atomic::{AtomicBool, Ordering};++ use futures::StreamExt;use switchyard_libsy::LlmTarget;use switchyard_protocol::{- LlmResponse, Response, RoutedLlmClient, Signals, completion_text, prompt_text,- text_request, text_response,+ LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Response, Role,+ RoutedLlmClient, SamplingParams, Signals, ToolChoice, ToolDefinition, completion_text,+ prompt_text, text_request, text_response,};+ use tokio::sync::Semaphore;/// Mock client that answers candidate calls with `answer from {model}` and,/// for the judge model, returns the 1-based number of the response whose@@ -537,18 +626,7 @@ mod tests {prefer: prefer.to_string(),calls: Arc::clone(&calls),}) as Arc<dyn RoutedLlmClient>;- let target = |name: &str| LlmTarget {- semantic_name: name.to_string(),- llm_client: Some(client.clone()),- };- let mut targets: Vec<LlmTarget> = candidates.iter().map(|n| target(n)).collect();- targets.push(target(judge));- let algo = EnsembleOrchAlgo::new(- candidates.iter().map(|s| s.to_string()).collect(),- judge.to_string(),- exploration_turns,- LlmTargetSet::new(targets),- );+ let algo = algo_with_client(candidates, judge, exploration_turns, client);(algo, calls)}@@ -915,4 +993,447 @@ mod tests {assert_eq!(completion_b, "answer from b/model");Ok(())}++ /// Records the `LlmRequest` of every candidate call; answers candidates with+ /// `answer from {model}` and the judge with "1".+ struct CapturingClient {+ judge_model: String,+ candidate_requests: Arc<Mutex<Vec<LlmRequest>>>,+ }++ #[async_trait]+ impl RoutedLlmClient for CapturingClient {+ async fn call(+ &self,+ _ctx: Context,+ request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ let name = decision.selected_model().to_string();+ let completion = if name == self.judge_model {+ "1".to_string()+ } else {+ self.candidate_requests+ .lock()+ .push(request.llm_request.clone());+ format!("answer from {name}")+ };+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, completion)),+ metadata: None,+ })+ }+ }++ #[tokio::test]+ async fn candidate_and_committed_calls_preserve_the_full_request() -> Result<()> {+ // A request a single text prompt would flatten away: system + user+ // messages, a tool, a tool_choice, non-default sampling, and stream = true.+ let original = LlmRequest {+ model: Some("auto".to_string()),+ messages: vec![+ Message::text(Role::System, "be terse"),+ Message::text(Role::User, "add 2 and 2"),+ ],+ tools: vec![ToolDefinition {+ name: "calc".to_string(),+ description: Some("do math".to_string()),+ parameters: serde_json::json!({"type": "object"}),+ strict: Some(true),+ }],+ tool_choice: Some(ToolChoice::Required),+ sampling: SamplingParams {+ temperature: Some(0.3),+ ..Default::default()+ },+ stream: true,+ ..Default::default()+ };+ let candidate_requests = Arc::new(Mutex::new(Vec::new()));+ let client = Arc::new(CapturingClient {+ judge_model: "judge/haiku".to_string(),+ candidate_requests: Arc::clone(&candidate_requests),+ }) as Arc<dyn RoutedLlmClient>;+ let session = orch(algo_with_client(+ &["a/model", "b/model"],+ "judge/haiku",+ 1,+ client,+ ));+ for _ in 0..2 {+ session+ .clone()+ .run(+ Context::default(),+ Request {+ llm_request: original.clone(),+ raw_request: None,+ metadata: None,+ },+ )+ .await?;+ }++ // The first request reaches both candidates. The second request reaches+ // the committed model, and all three calls keep the original fields.+ let captured = candidate_requests.lock();+ assert_eq!(captured.len(), 3);+ for request in captured.iter() {+ assert_eq!(request, &original);+ }+ Ok(())+ }++ /// Candidate calls stream their answer as text deltas; the judge records the+ /// prompt it saw and picks the response mentioning `prefer`.+ struct StreamingCandidatesClient {+ judge_model: String,+ prefer: String,+ judge_prompts: Arc<Mutex<Vec<String>>>,+ }++ #[async_trait]+ impl RoutedLlmClient for StreamingCandidatesClient {+ async fn call(+ &self,+ _ctx: Context,+ request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ let name = decision.selected_model().to_string();+ if name == self.judge_model {+ let prompt = prompt_text(&request.llm_request);+ self.judge_prompts.lock().push(prompt.clone());+ return Ok(Response {+ llm_response: LlmResponse::Agg(text_response(+ None,+ judge_pick(&prompt, &self.prefer),+ )),+ metadata: None,+ });+ }+ // Candidate answers arrive as a token stream, not a buffered response.+ let answer = format!("answer from {name}");+ let stream = futures::stream::iter(+ [LlmResponseStreamEvent::from(LlmResponseChunk::TextDelta {+ index: 0,+ text: answer,+ })]+ .into_iter()+ .map(Ok),+ )+ .boxed();+ Ok(Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ })+ }+ }++ #[tokio::test]+ async fn streamed_candidates_are_judged_on_their_real_text() -> Result<()> {+ let judge_prompts = Arc::new(Mutex::new(Vec::new()));+ let client = Arc::new(StreamingCandidatesClient {+ judge_model: "judge/haiku".to_string(),+ prefer: "b/model".to_string(),+ judge_prompts: Arc::clone(&judge_prompts),+ }) as Arc<dyn RoutedLlmClient>;+ let algo = algo_with_client(&["a/model", "b/model"], "judge/haiku", 100, client);+ let (_, response) = orch(algo)+ .run(Context::default(), request("solve it"))+ .await?;++ // The judge saw both streamed candidate answers, so it could choose on+ // content rather than fall open to the first response on empty text.+ let prompt = judge_prompts.lock().first().cloned().unwrap_or_default();+ assert!(+ prompt.contains("answer from a/model"),+ "judge prompt: {prompt}"+ );+ assert!(+ prompt.contains("answer from b/model"),+ "judge prompt: {prompt}"+ );+ // The winner is returned as buffered text, not lost with its stream.+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from b/model"+ );+ Ok(())+ }++ /// Candidate calls announce arrival then block on `gate` until the test opens+ /// it; the judge answers immediately, preferring `prefer`.+ struct GatedCandidateClient {+ judge_model: String,+ prefer: String,+ calls: Arc<Mutex<Vec<String>>>,+ arrivals: Arc<Semaphore>,+ gate: Arc<Semaphore>,+ }++ #[async_trait]+ impl RoutedLlmClient for GatedCandidateClient {+ async fn call(+ &self,+ _ctx: Context,+ request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ let name = decision.selected_model().to_string();+ self.calls.lock().push(name.clone());+ if name == self.judge_model {+ let completion = judge_pick(&prompt_text(&request.llm_request), &self.prefer);+ return Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, completion)),+ metadata: None,+ });+ }+ // Announce this candidate has arrived, then wait for the test to release.+ self.arrivals.add_permits(1);+ if let Ok(permit) = self.gate.acquire().await {+ permit.forget();+ }+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {name}"))),+ metadata: None,+ })+ }+ }++ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]+ async fn concurrent_requests_do_not_exceed_the_exploration_budget() -> Result<()> {+ let calls = Arc::new(Mutex::new(Vec::<String>::new()));+ let arrivals = Arc::new(Semaphore::new(0));+ let gate = Arc::new(Semaphore::new(0));+ let client = Arc::new(GatedCandidateClient {+ judge_model: "judge/haiku".to_string(),+ prefer: "b/model".to_string(),+ calls: Arc::clone(&calls),+ arrivals: Arc::clone(&arrivals),+ gate: Arc::clone(&gate),+ }) as Arc<dyn RoutedLlmClient>;+ // exploration_turns = 1: exactly one turn may explore before committing.+ let algo: Arc<dyn Algorithm> = Arc::new(algo_with_client(+ &["a/model", "b/model"],+ "judge/haiku",+ 1,+ client,+ ));++ // Request A reserves the only slot and fans out; its candidate calls block.+ let handle_a = tokio::spawn({+ let algo = algo.clone();+ async move { algo.run(Context::default(), request("A")).await }+ });+ // Wait for A's two candidates to reach the gate — A has reserved the slot.+ let arrived = arrivals+ .acquire_many(2)+ .await+ .map_err(|_| ensemble_error("arrivals semaphore closed"))?;+ arrived.forget();++ // Request B arrives while A is still exploring. With the budget reserved it+ // must wait for A, not start a second exploration or commit early.+ let handle_b = tokio::spawn({+ let algo = algo.clone();+ async move { algo.run(Context::default(), request("B")).await }+ });++ // Open the gate for every remaining call: A's two candidates, and B's later+ // committed route call. The reservation logic, not the gate, is what bounds+ // exploration — so if B wrongly fanned out, its candidate calls would pass+ // here and the counts below would catch it.+ gate.add_permits(100);+ handle_a.await??;+ let (b_trace, _) = handle_b.await??;++ let calls = calls.lock();+ // Exactly one exploration ran: the judge was consulted once, and B did not+ // fan out to a/model (A's single exploration is the only fan-out).+ assert_eq!(calls.iter().filter(|c| *c == "judge/haiku").count(), 1);+ assert_eq!(calls.iter().filter(|c| *c == "a/model").count(), 1);+ // b/model: A's exploration fan-out plus B's committed route to the winner.+ assert_eq!(calls.iter().filter(|c| *c == "b/model").count(), 2);++ // B committed from A's complete tally: b/model won A's turn, ruling out a+ // premature commit on the empty tally (which would pick the first+ // candidate, a/model).+ let decision = as_ensemble(+ b_trace+ .last()+ .ok_or_else(|| ensemble_error("B produced no decision"))?,+ )?;+ assert_eq!(decision.phase, EnsemblePhase::Committed);+ assert_eq!(decision.selected_model, "b/model");+ Ok(())+ }++ /// Fails every candidate call while `fail` is set and answers normally once+ /// the test clears it; the judge always answers "1".+ struct FlakyCandidateClient {+ judge_model: String,+ fail: Arc<AtomicBool>,+ }++ #[async_trait]+ impl RoutedLlmClient for FlakyCandidateClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ let name = decision.selected_model().to_string();+ if name == self.judge_model {+ return Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, "1")),+ metadata: None,+ });+ }+ if self.fail.load(Ordering::SeqCst) {+ return Err(switchyard_protocol::LlmClientError::General(+ "candidate unavailable".to_string(),+ ));+ }+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {name}"))),+ metadata: None,+ })+ }+ }++ #[tokio::test]+ async fn a_failed_exploration_frees_the_slot_for_the_next_request() -> Result<()> {+ let fail = Arc::new(AtomicBool::new(true));+ let client = Arc::new(FlakyCandidateClient {+ judge_model: "judge/haiku".to_string(),+ fail: Arc::clone(&fail),+ }) as Arc<dyn RoutedLlmClient>;+ // exploration_turns = 1: a single slot serves the whole session.+ let session = orch(algo_with_client(+ &["a/model", "b/model"],+ "judge/haiku",+ 1,+ client,+ ));++ // Request A reserves the only slot and fails every candidate, so its turn+ // errors — and must return the slot instead of spending the budget.+ assert!(+ session+ .clone()+ .run(Context::default(), request("A"))+ .await+ .is_err()+ );++ // With the slot freed, request B can still explore. A leaked reservation+ // would park B forever waiting for the exploration budget to drain, so the+ // timeout turns that deadlock into a test failure instead of a hang.+ fail.store(false, Ordering::SeqCst);+ let (trace, _) = tokio::time::timeout(+ std::time::Duration::from_secs(5),+ session.run(Context::default(), request("B")),+ )+ .await+ .map_err(|_| ensemble_error("request B hung: the failed turn leaked its slot"))??;+ let decision = as_ensemble(+ trace+ .last()+ .ok_or_else(|| ensemble_error("B produced no decision"))?,+ )?;+ assert_eq!(decision.phase, EnsemblePhase::Winner);+ Ok(())+ }++ /// Streams one candidate answer that fails partway through, so aggregating it+ /// errors; answers other candidates normally and records judge calls.+ struct BrokenStreamClient {+ judge_model: String,+ broken: String,+ judge_calls: Arc<Mutex<u32>>,+ }++ #[async_trait]+ impl RoutedLlmClient for BrokenStreamClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ let name = decision.selected_model().to_string();+ if name == self.judge_model {+ *self.judge_calls.lock() += 1;+ return Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, "1")),+ metadata: None,+ });+ }+ if name == self.broken {+ // A stream that yields text then an error: draining it to an+ // aggregate fails, so this candidate is excluded from judging.+ let stream = futures::stream::iter([+ Ok(LlmResponseStreamEvent::from(LlmResponseChunk::TextDelta {+ index: 0,+ text: "partial".to_string(),+ })),+ Err(switchyard_protocol::LlmClientError::General(+ "stream broke".to_string(),+ )),+ ])+ .boxed();+ return Ok(Response {+ llm_response: LlmResponse::Stream(stream),+ metadata: None,+ });+ }+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {name}"))),+ metadata: None,+ })+ }+ }++ #[tokio::test]+ async fn a_candidate_whose_stream_fails_is_excluded_not_fatal() -> Result<()> {+ let judge_calls = Arc::new(Mutex::new(0u32));+ let client = Arc::new(BrokenStreamClient {+ judge_model: "judge/haiku".to_string(),+ broken: "a/model".to_string(),+ judge_calls: Arc::clone(&judge_calls),+ }) as Arc<dyn RoutedLlmClient>;+ let (trace, response) = orch(algo_with_client(+ &["a/model", "b/model"],+ "judge/haiku",+ 100,+ client,+ ))+ .run(Context::default(), request("solve it"))+ .await?;++ // b/model is the lone survivor, so it wins with no judge call...+ assert_eq!(*judge_calls.lock(), 0);+ // ...and its buffered text is returned, not lost with a/model's stream.+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from b/model"+ );+ let winner = as_ensemble(+ trace+ .last()+ .ok_or_else(|| ensemble_error("no winner decision"))?,+ )?;+ assert_eq!(winner.selected_model, "b/model");+ Ok(())+ }}