Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(examples): forward full ensemble requests and cap concurrent explorations (#202)

Elyas Mehtabuddin8d47c2f3e52a
diff --git a/crates/libsy/examples/ensemble.rs b/crates/libsy/examples/ensemble.rs
index 27520403f..7f96b1d8f 100644
--- a/crates/libsy/examples/ensemble.rs
+++ b/crates/libsy/examples/ensemble.rs
@@ -13,10 +13,11 @@
//! NVIDIA_API_KEY=... cargo run -p switchyard-libsy --example ensemble
//!
//! Unlike the reference routers, this algorithm is **stateful**: the win tally,
-//! turn counter, and committed choice live behind a [`parking_lot::Mutex`] so one
-//! shared `&self` can serve a session's requests concurrently (see the
-//! `Algorithm` docs). In a proxy setup one `EnsembleOrchAlgo` is created per session,
-//! so this state is per-session.
+//! the claimed exploration count, and the committed choice live behind a
+//! [`parking_lot::Mutex`] so one shared `&self` can serve a session's
+//! requests concurrently (see the `Algorithm` docs); a [`tokio::sync::Notify`]
+//! wakes a request waiting for the exploration budget to finish. In a proxy setup
+//! one `EnsembleOrchAlgo` is created per session, so this state is per-session.
use std::collections::BTreeMap;
use std::sync::Arc;
@@ -24,11 +25,13 @@ use std::sync::Arc;
use async_trait::async_trait;
use parking_lot::Mutex;
+use tokio::sync::Notify;
+
use switchyard_libsy::{Algorithm, Driver, LibsyError, LlmTarget, LlmTargetSet, Result};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
use switchyard_protocol::{
- Context, Decision, Request, Response, RoutedLlmClient, completion_text, prompt_text,
- text_request,
+ Context, Decision, LlmResponse, Request, Response, RoutedLlmClient, completion_text,
+ prompt_text, text_request,
};
const CANDIDATE_MODELS: [&str; 3] = [
@@ -95,12 +98,44 @@ impl Decision for EnsembleDecision {
struct EnsembleState {
/// Judge-win count per candidate model.
wins: BTreeMap<String, u64>,
- /// Completed ensemble (exploration) turns.
- turns: u64,
+ /// Running or completed exploration turns. Failed turns return their slot.
+ used_slots: u64,
/// The model committed to once exploration is over; `None` while exploring.
committed: Option<String>,
}
+/// How one request is served: straight to the committed model, or as a fresh
+/// exploration turn (a slot was reserved for it under the lock).
+enum RoutePlan {
+ Committed(String),
+ Explore,
+}
+
+/// Holds a reserved exploration slot for the length of one turn and returns it if
+/// the turn is dropped before it tallies a winner — the run future aborted on a
+/// client disconnect, a panic, or an error. A turn that completes calls
+/// [`disarm`](ReservationGuard::disarm) and keeps its now-counted slot.
+struct ReservationGuard<'a> {
+ algo: &'a EnsembleOrchAlgo,
+ armed: bool,
+}
+
+impl ReservationGuard<'_> {
+ /// Give up the claim on the slot: the turn completed and counted it, so it
+ /// must not be released.
+ fn disarm(&mut self) {
+ self.armed = false;
+ }
+}
+
+impl Drop for ReservationGuard<'_> {
+ fn drop(&mut self) {
+ if self.armed {
+ self.algo.release_reservation();
+ }
+ }
+}
+
/// Ensemble router: fan out to candidates, judge the best, then commit.
pub struct EnsembleOrchAlgo {
candidate_models: Vec<String>,
@@ -110,6 +145,9 @@ pub struct EnsembleOrchAlgo {
exploration_turns: u64,
target_set: LlmTargetSet,
state: Mutex<EnsembleState>,
+ /// Notified whenever a reserved exploration finishes, so a request waiting
+ /// for the exploration budget to drain can recheck and commit.
+ explorations_done: Notify,
}
impl EnsembleOrchAlgo {
@@ -132,30 +170,59 @@ impl EnsembleOrchAlgo {
target_set,
state: Mutex::new(EnsembleState {
wins: BTreeMap::new(),
- turns: 0,
+ used_slots: 0,
committed: None,
}),
+ explorations_done: Notify::new(),
}
}
- /// If exploration is over, return the committed model (committing lazily on
- /// the first post-exploration request). `None` means keep ensembling.
+ /// Decide how to serve one request, reserving an exploration slot under the
+ /// lock before any async work begins. This caps concurrent explorations at
+ /// `exploration_turns` and commits only from a complete tally:
///
- /// The lock is taken and dropped here, never across an `await`. Under
- /// concurrency two requests may both commit; they pick the same model from
- /// the same tally (with a stable tie-break), so the result is identical.
- fn resolve_committed(&self) -> Result<Option<String>> {
- let mut state = self.state.lock();
- if let Some(model) = &state.committed {
- return Ok(Some(model.clone()));
- }
- // `exploration_turns == 0` keeps the algorithm in ensemble mode forever.
- if self.exploration_turns > 0 && state.turns >= self.exploration_turns {
- let best = self.pick_best(&state.wins)?;
- state.committed = Some(best.clone());
- return Ok(Some(best));
+ /// - already committed → route to that model;
+ /// - a slot is free (`used_slots < exploration_turns`, or unlimited when
+ /// `exploration_turns == 0`) → reserve it and explore;
+ /// - the budget is fully reserved but some explorations are still running →
+ /// wait for them rather than commit early or start an extra turn;
+ /// - the budget is fully reserved and every reservation has been tallied →
+ /// commit to the winningest model.
+ ///
+ /// The lock is only held for the synchronous decision; the wait happens after
+ /// it is released, with the wakeup registered first so no completion is missed.
+ async fn plan_route(&self) -> Result<RoutePlan> {
+ loop {
+ let wait = self.explorations_done.notified();
+ tokio::pin!(wait);
+ {
+ let mut state = self.state.lock();
+ if let Some(model) = &state.committed {
+ return Ok(RoutePlan::Committed(model.clone()));
+ }
+ // `exploration_turns == 0` keeps the algorithm ensembling forever.
+ if self.exploration_turns == 0 || state.used_slots < self.exploration_turns {
+ state.used_slots += 1;
+ return Ok(RoutePlan::Explore);
+ }
+ if state.wins.values().sum::<u64>() >= self.exploration_turns {
+ let best = self.pick_best(&state.wins)?;
+ state.committed = Some(best.clone());
+ return Ok(RoutePlan::Committed(best));
+ }
+ // Register for the next completion before dropping the lock so a
+ // notify between here and the await below is not lost.
+ wait.as_mut().enable();
+ }
+ wait.await;
}
- Ok(None)
+ }
+
+ /// Return an exploration slot whose turn failed without tallying a winner, so
+ /// a waiting request can take it and the budget is not spent by a failed turn.
+ fn release_reservation(&self) {
+ self.state.lock().used_slots -= 1;
+ self.explorations_done.notify_waiters();
}
/// The candidate with the most judge-wins, breaking ties toward the earlier
@@ -191,39 +258,23 @@ impl EnsembleOrchAlgo {
selected_model: model.clone(),
phase: EnsemblePhase::Committed,
});
- // The agent's inbound name rides through; the committed model is on the
- // decision, not stamped onto the request.
- let routed = Request {
- llm_request: text_request(
- request.llm_request.model.clone(),
- prompt_text(&request.llm_request),
- ),
- raw_request: request.raw_request,
- metadata: request.metadata,
- };
+ // Forward the caller's complete request unchanged; the committed model is
+ // carried on the decision, not stamped onto the request.
let response = driver
- .call_llm_target(ctx, &target, routed, decision.clone())
+ .call_llm_target(ctx, &target, request, decision.clone())
.await?;
Ok((vec![decision], response))
}
- /// One exploration turn: fan out to every candidate, judge the survivors,
- /// tally the winner, and return its response.
- async fn ensemble_turn(
+ /// Call all candidates concurrently and buffer successful responses. A call
+ /// or stream error removes only that candidate from the turn.
+ async fn fan_out_candidates(
&self,
driver: &Driver,
- ctx: Context,
- request: Request,
- ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
- let user_prompt = prompt_text(&request.llm_request);
- // The agent's inbound name rides through every sub-call unchanged; the model
- // each call hits is carried by its decision, not stamped onto the request.
- let inbound = request.llm_request.model.clone();
-
- // Fan out to all candidates concurrently with the same user prompt. Each
- // call is annotated with its own candidate decision so the caller can see
- // which model an offloaded call targets.
- let mut candidate_decisions: Vec<Arc<dyn Decision>> = Vec::new();
+ ctx: &Context,
+ request: &Request,
+ ) -> Result<(Vec<Arc<dyn Decision>>, Vec<(String, Response)>)> {
+ let mut decisions: Vec<Arc<dyn Decision>> = Vec::new();
let mut calls = Vec::new();
for model in &self.candidate_models {
let target = self.target_set.get_target(model)?;
@@ -232,33 +283,53 @@ impl EnsembleOrchAlgo {
reasoning: format!("ensemble candidate '{model}'"),
phase: EnsemblePhase::Candidate,
});
- candidate_decisions.push(decision.clone());
- let call_request = Request {
- llm_request: text_request(inbound.clone(), user_prompt.clone()),
- raw_request: request.raw_request.clone(),
- metadata: request.metadata.clone(),
- };
+ decisions.push(decision.clone());
+ let request = request.clone();
let model = model.clone();
let ctx = ctx.clone();
calls.push(async move {
- (
+ // Drain each stream inside its candidate future so slow streams do
+ // not block the other candidates from making progress.
+ let Response {
+ llm_response,
+ metadata,
+ } = driver
+ .call_llm_target(ctx, &target, request, decision)
+ .await
+ .ok()?;
+ let llm_response = LlmResponse::Agg(llm_response.into_agg().await.ok()?);
+ Some((
model,
- driver
- .call_llm_target(ctx, &target, call_request, decision)
- .await,
- )
+ Response {
+ llm_response,
+ metadata,
+ },
+ ))
});
}
- let results = futures::future::join_all(calls).await;
-
- // Keep only successful responses, preserving candidate order. A failed
- // candidate is simply excluded from judging rather than failing the turn.
- let mut survivors: Vec<(String, Response)> = Vec::new();
- for (model, result) in results {
- if let Ok(response) = result {
- survivors.push((model, response));
- }
- }
+
+ let survivors = futures::future::join_all(calls)
+ .await
+ .into_iter()
+ .flatten()
+ .collect();
+ Ok((decisions, survivors))
+ }
+
+ /// One exploration turn: fan out to every candidate, judge the survivors,
+ /// tally the winner, and return its response.
+ async fn ensemble_turn(
+ &self,
+ driver: &Driver,
+ ctx: Context,
+ request: Request,
+ ) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
+ let user_prompt = prompt_text(&request.llm_request);
+ // Keep the caller's model ID; each decision selects the actual target.
+ let inbound = request.llm_request.model.clone();
+
+ let (candidate_decisions, survivors) =
+ self.fan_out_candidates(driver, &ctx, &request).await?;
if survivors.is_empty() {
return Err(ensemble_error("all candidate calls failed"));
}
@@ -279,7 +350,7 @@ impl EnsembleOrchAlgo {
phase: EnsemblePhase::Judge,
});
let judge_request = Request {
- llm_request: text_request(inbound.clone(), judge_prompt),
+ llm_request: text_request(inbound, judge_prompt),
raw_request: request.raw_request.clone(),
metadata: request.metadata.clone(),
};
@@ -302,13 +373,13 @@ impl EnsembleOrchAlgo {
(model, response, Some(judge_decision))
};
- // Record the win and advance the turn counter under the lock (not held
- // across any await).
- {
- let mut state = self.state.lock();
- *state.wins.entry(winner_model.clone()).or_insert(0) += 1;
- state.turns += 1;
- }
+ *self
+ .state
+ .lock()
+ .wins
+ .entry(winner_model.clone())
+ .or_insert(0) += 1;
+ self.explorations_done.notify_waiters();
let winner_decision: Arc<dyn Decision> = Arc::new(EnsembleDecision {
reasoning: format!("judge selected '{winner_model}' as best response"),
@@ -383,17 +454,30 @@ impl Algorithm for EnsembleOrchAlgo {
driver: Driver,
request: Request,
) -> Result<Response> {
- // Fast path: exploration is over — route straight to the committed model;
- // otherwise run a full ensemble turn. Both return a decision trace plus the
- // final response.
- let (trace, response) = if let Some(model) = self.resolve_committed()? {
- self.route_committed(&driver, ctx.clone(), request, model)
- .await?
- } else {
- self.ensemble_turn(&driver, ctx.clone(), request).await?
+ // Reserve the route (committed model or a fresh exploration slot) before any
+ // async work, then serve it. A reserved exploration that fails releases its
+ // slot so the budget is not spent by a failed turn.
+ let (trace, response) = match self.plan_route().await? {
+ RoutePlan::Committed(model) => {
+ self.route_committed(&driver, ctx.clone(), request, model)
+ .await?
+ }
+ RoutePlan::Explore => {
+ // Hold the reserved slot across the turn: an abort or panic
+ // mid-turn returns it through the guard's `Drop`, and an error
+ // returns it on the way out. A turn that completes disarms the
+ // guard and keeps its now-counted slot.
+ let mut guard = ReservationGuard {
+ algo: &self,
+ armed: true,
+ };
+ let result = self.ensemble_turn(&driver, ctx.clone(), request).await?;
+ guard.disarm();
+ result
+ }
};
// Publish the trace to the stream (candidate..., judge?, winner). The
- // candidate decisions also rode along on their offloaded `CallLlm` steps.
+ // candidate decisions were also attached to their `CallLlm` steps.
for decision in trace {
driver.info(ctx.clone(), decision).await?;
}
@@ -467,11 +551,16 @@ async fn main() -> Result<()> {
#[cfg(test)]
mod tests {
use super::*;
+ use std::sync::atomic::{AtomicBool, Ordering};
+
+ use futures::StreamExt;
use switchyard_libsy::LlmTarget;
use switchyard_protocol::{
- LlmResponse, Response, RoutedLlmClient, Signals, completion_text, prompt_text,
- text_request, text_response,
+ LlmRequest, LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, Message, Response, Role,
+ RoutedLlmClient, SamplingParams, Signals, ToolChoice, ToolDefinition, completion_text,
+ prompt_text, text_request, text_response,
};
+ use tokio::sync::Semaphore;
/// Mock client that answers candidate calls with `answer from {model}` and,
/// for the judge model, returns the 1-based number of the response whose
@@ -537,18 +626,7 @@ mod tests {
prefer: prefer.to_string(),
calls: Arc::clone(&calls),
}) as Arc<dyn RoutedLlmClient>;
- let target = |name: &str| LlmTarget {
- semantic_name: name.to_string(),
- llm_client: Some(client.clone()),
- };
- let mut targets: Vec<LlmTarget> = candidates.iter().map(|n| target(n)).collect();
- targets.push(target(judge));
- let algo = EnsembleOrchAlgo::new(
- candidates.iter().map(|s| s.to_string()).collect(),
- judge.to_string(),
- exploration_turns,
- LlmTargetSet::new(targets),
- );
+ let algo = algo_with_client(candidates, judge, exploration_turns, client);
(algo, calls)
}
@@ -915,4 +993,447 @@ mod tests {
assert_eq!(completion_b, "answer from b/model");
Ok(())
}
+
+ /// Records the `LlmRequest` of every candidate call; answers candidates with
+ /// `answer from {model}` and the judge with "1".
+ struct CapturingClient {
+ judge_model: String,
+ candidate_requests: Arc<Mutex<Vec<LlmRequest>>>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for CapturingClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ let name = decision.selected_model().to_string();
+ let completion = if name == self.judge_model {
+ "1".to_string()
+ } else {
+ self.candidate_requests
+ .lock()
+ .push(request.llm_request.clone());
+ format!("answer from {name}")
+ };
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
+ metadata: None,
+ })
+ }
+ }
+
+ #[tokio::test]
+ async fn candidate_and_committed_calls_preserve_the_full_request() -> Result<()> {
+ // A request a single text prompt would flatten away: system + user
+ // messages, a tool, a tool_choice, non-default sampling, and stream = true.
+ let original = LlmRequest {
+ model: Some("auto".to_string()),
+ messages: vec![
+ Message::text(Role::System, "be terse"),
+ Message::text(Role::User, "add 2 and 2"),
+ ],
+ tools: vec![ToolDefinition {
+ name: "calc".to_string(),
+ description: Some("do math".to_string()),
+ parameters: serde_json::json!({"type": "object"}),
+ strict: Some(true),
+ }],
+ tool_choice: Some(ToolChoice::Required),
+ sampling: SamplingParams {
+ temperature: Some(0.3),
+ ..Default::default()
+ },
+ stream: true,
+ ..Default::default()
+ };
+ let candidate_requests = Arc::new(Mutex::new(Vec::new()));
+ let client = Arc::new(CapturingClient {
+ judge_model: "judge/haiku".to_string(),
+ candidate_requests: Arc::clone(&candidate_requests),
+ }) as Arc<dyn RoutedLlmClient>;
+ let session = orch(algo_with_client(
+ &["a/model", "b/model"],
+ "judge/haiku",
+ 1,
+ client,
+ ));
+ for _ in 0..2 {
+ session
+ .clone()
+ .run(
+ Context::default(),
+ Request {
+ llm_request: original.clone(),
+ raw_request: None,
+ metadata: None,
+ },
+ )
+ .await?;
+ }
+
+ // The first request reaches both candidates. The second request reaches
+ // the committed model, and all three calls keep the original fields.
+ let captured = candidate_requests.lock();
+ assert_eq!(captured.len(), 3);
+ for request in captured.iter() {
+ assert_eq!(request, &original);
+ }
+ Ok(())
+ }
+
+ /// Candidate calls stream their answer as text deltas; the judge records the
+ /// prompt it saw and picks the response mentioning `prefer`.
+ struct StreamingCandidatesClient {
+ judge_model: String,
+ prefer: String,
+ judge_prompts: Arc<Mutex<Vec<String>>>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for StreamingCandidatesClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ let name = decision.selected_model().to_string();
+ if name == self.judge_model {
+ let prompt = prompt_text(&request.llm_request);
+ self.judge_prompts.lock().push(prompt.clone());
+ return Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(
+ None,
+ judge_pick(&prompt, &self.prefer),
+ )),
+ metadata: None,
+ });
+ }
+ // Candidate answers arrive as a token stream, not a buffered response.
+ let answer = format!("answer from {name}");
+ let stream = futures::stream::iter(
+ [LlmResponseStreamEvent::from(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: answer,
+ })]
+ .into_iter()
+ .map(Ok),
+ )
+ .boxed();
+ Ok(Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ })
+ }
+ }
+
+ #[tokio::test]
+ async fn streamed_candidates_are_judged_on_their_real_text() -> Result<()> {
+ let judge_prompts = Arc::new(Mutex::new(Vec::new()));
+ let client = Arc::new(StreamingCandidatesClient {
+ judge_model: "judge/haiku".to_string(),
+ prefer: "b/model".to_string(),
+ judge_prompts: Arc::clone(&judge_prompts),
+ }) as Arc<dyn RoutedLlmClient>;
+ let algo = algo_with_client(&["a/model", "b/model"], "judge/haiku", 100, client);
+ let (_, response) = orch(algo)
+ .run(Context::default(), request("solve it"))
+ .await?;
+
+ // The judge saw both streamed candidate answers, so it could choose on
+ // content rather than fall open to the first response on empty text.
+ let prompt = judge_prompts.lock().first().cloned().unwrap_or_default();
+ assert!(
+ prompt.contains("answer from a/model"),
+ "judge prompt: {prompt}"
+ );
+ assert!(
+ prompt.contains("answer from b/model"),
+ "judge prompt: {prompt}"
+ );
+ // The winner is returned as buffered text, not lost with its stream.
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from b/model"
+ );
+ Ok(())
+ }
+
+ /// Candidate calls announce arrival then block on `gate` until the test opens
+ /// it; the judge answers immediately, preferring `prefer`.
+ struct GatedCandidateClient {
+ judge_model: String,
+ prefer: String,
+ calls: Arc<Mutex<Vec<String>>>,
+ arrivals: Arc<Semaphore>,
+ gate: Arc<Semaphore>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for GatedCandidateClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ let name = decision.selected_model().to_string();
+ self.calls.lock().push(name.clone());
+ if name == self.judge_model {
+ let completion = judge_pick(&prompt_text(&request.llm_request), &self.prefer);
+ return Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, completion)),
+ metadata: None,
+ });
+ }
+ // Announce this candidate has arrived, then wait for the test to release.
+ self.arrivals.add_permits(1);
+ if let Ok(permit) = self.gate.acquire().await {
+ permit.forget();
+ }
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {name}"))),
+ metadata: None,
+ })
+ }
+ }
+
+ #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+ async fn concurrent_requests_do_not_exceed_the_exploration_budget() -> Result<()> {
+ let calls = Arc::new(Mutex::new(Vec::<String>::new()));
+ let arrivals = Arc::new(Semaphore::new(0));
+ let gate = Arc::new(Semaphore::new(0));
+ let client = Arc::new(GatedCandidateClient {
+ judge_model: "judge/haiku".to_string(),
+ prefer: "b/model".to_string(),
+ calls: Arc::clone(&calls),
+ arrivals: Arc::clone(&arrivals),
+ gate: Arc::clone(&gate),
+ }) as Arc<dyn RoutedLlmClient>;
+ // exploration_turns = 1: exactly one turn may explore before committing.
+ let algo: Arc<dyn Algorithm> = Arc::new(algo_with_client(
+ &["a/model", "b/model"],
+ "judge/haiku",
+ 1,
+ client,
+ ));
+
+ // Request A reserves the only slot and fans out; its candidate calls block.
+ let handle_a = tokio::spawn({
+ let algo = algo.clone();
+ async move { algo.run(Context::default(), request("A")).await }
+ });
+ // Wait for A's two candidates to reach the gate — A has reserved the slot.
+ let arrived = arrivals
+ .acquire_many(2)
+ .await
+ .map_err(|_| ensemble_error("arrivals semaphore closed"))?;
+ arrived.forget();
+
+ // Request B arrives while A is still exploring. With the budget reserved it
+ // must wait for A, not start a second exploration or commit early.
+ let handle_b = tokio::spawn({
+ let algo = algo.clone();
+ async move { algo.run(Context::default(), request("B")).await }
+ });
+
+ // Open the gate for every remaining call: A's two candidates, and B's later
+ // committed route call. The reservation logic, not the gate, is what bounds
+ // exploration — so if B wrongly fanned out, its candidate calls would pass
+ // here and the counts below would catch it.
+ gate.add_permits(100);
+ handle_a.await??;
+ let (b_trace, _) = handle_b.await??;
+
+ let calls = calls.lock();
+ // Exactly one exploration ran: the judge was consulted once, and B did not
+ // fan out to a/model (A's single exploration is the only fan-out).
+ assert_eq!(calls.iter().filter(|c| *c == "judge/haiku").count(), 1);
+ assert_eq!(calls.iter().filter(|c| *c == "a/model").count(), 1);
+ // b/model: A's exploration fan-out plus B's committed route to the winner.
+ assert_eq!(calls.iter().filter(|c| *c == "b/model").count(), 2);
+
+ // B committed from A's complete tally: b/model won A's turn, ruling out a
+ // premature commit on the empty tally (which would pick the first
+ // candidate, a/model).
+ let decision = as_ensemble(
+ b_trace
+ .last()
+ .ok_or_else(|| ensemble_error("B produced no decision"))?,
+ )?;
+ assert_eq!(decision.phase, EnsemblePhase::Committed);
+ assert_eq!(decision.selected_model, "b/model");
+ Ok(())
+ }
+
+ /// Fails every candidate call while `fail` is set and answers normally once
+ /// the test clears it; the judge always answers "1".
+ struct FlakyCandidateClient {
+ judge_model: String,
+ fail: Arc<AtomicBool>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for FlakyCandidateClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ let name = decision.selected_model().to_string();
+ if name == self.judge_model {
+ return Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, "1")),
+ metadata: None,
+ });
+ }
+ if self.fail.load(Ordering::SeqCst) {
+ return Err(switchyard_protocol::LlmClientError::General(
+ "candidate unavailable".to_string(),
+ ));
+ }
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {name}"))),
+ metadata: None,
+ })
+ }
+ }
+
+ #[tokio::test]
+ async fn a_failed_exploration_frees_the_slot_for_the_next_request() -> Result<()> {
+ let fail = Arc::new(AtomicBool::new(true));
+ let client = Arc::new(FlakyCandidateClient {
+ judge_model: "judge/haiku".to_string(),
+ fail: Arc::clone(&fail),
+ }) as Arc<dyn RoutedLlmClient>;
+ // exploration_turns = 1: a single slot serves the whole session.
+ let session = orch(algo_with_client(
+ &["a/model", "b/model"],
+ "judge/haiku",
+ 1,
+ client,
+ ));
+
+ // Request A reserves the only slot and fails every candidate, so its turn
+ // errors — and must return the slot instead of spending the budget.
+ assert!(
+ session
+ .clone()
+ .run(Context::default(), request("A"))
+ .await
+ .is_err()
+ );
+
+ // With the slot freed, request B can still explore. A leaked reservation
+ // would park B forever waiting for the exploration budget to drain, so the
+ // timeout turns that deadlock into a test failure instead of a hang.
+ fail.store(false, Ordering::SeqCst);
+ let (trace, _) = tokio::time::timeout(
+ std::time::Duration::from_secs(5),
+ session.run(Context::default(), request("B")),
+ )
+ .await
+ .map_err(|_| ensemble_error("request B hung: the failed turn leaked its slot"))??;
+ let decision = as_ensemble(
+ trace
+ .last()
+ .ok_or_else(|| ensemble_error("B produced no decision"))?,
+ )?;
+ assert_eq!(decision.phase, EnsemblePhase::Winner);
+ Ok(())
+ }
+
+ /// Streams one candidate answer that fails partway through, so aggregating it
+ /// errors; answers other candidates normally and records judge calls.
+ struct BrokenStreamClient {
+ judge_model: String,
+ broken: String,
+ judge_calls: Arc<Mutex<u32>>,
+ }
+
+ #[async_trait]
+ impl RoutedLlmClient for BrokenStreamClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ let name = decision.selected_model().to_string();
+ if name == self.judge_model {
+ *self.judge_calls.lock() += 1;
+ return Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, "1")),
+ metadata: None,
+ });
+ }
+ if name == self.broken {
+ // A stream that yields text then an error: draining it to an
+ // aggregate fails, so this candidate is excluded from judging.
+ let stream = futures::stream::iter([
+ Ok(LlmResponseStreamEvent::from(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: "partial".to_string(),
+ })),
+ Err(switchyard_protocol::LlmClientError::General(
+ "stream broke".to_string(),
+ )),
+ ])
+ .boxed();
+ return Ok(Response {
+ llm_response: LlmResponse::Stream(stream),
+ metadata: None,
+ });
+ }
+ Ok(Response {
+ llm_response: LlmResponse::Agg(text_response(None, format!("answer from {name}"))),
+ metadata: None,
+ })
+ }
+ }
+
+ #[tokio::test]
+ async fn a_candidate_whose_stream_fails_is_excluded_not_fatal() -> Result<()> {
+ let judge_calls = Arc::new(Mutex::new(0u32));
+ let client = Arc::new(BrokenStreamClient {
+ judge_model: "judge/haiku".to_string(),
+ broken: "a/model".to_string(),
+ judge_calls: Arc::clone(&judge_calls),
+ }) as Arc<dyn RoutedLlmClient>;
+ let (trace, response) = orch(algo_with_client(
+ &["a/model", "b/model"],
+ "judge/haiku",
+ 100,
+ client,
+ ))
+ .run(Context::default(), request("solve it"))
+ .await?;
+
+ // b/model is the lone survivor, so it wins with no judge call...
+ assert_eq!(*judge_calls.lock(), 0);
+ // ...and its buffered text is returned, not lost with a/model's stream.
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from b/model"
+ );
+ let winner = as_ensemble(
+ trace
+ .last()
+ .ok_or_else(|| ensemble_error("no winner decision"))?,
+ )?;
+ assert_eq!(winner.selected_model, "b/model");
+ Ok(())
+ }
}