Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy-llm-client): Move retry logic from libsy to libsy-llm-client (#431)
Graham King9ad6744b734b
diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.mdindex 207ea787b..d86dc7708 100644--- a/crates/libsy-llm-client/README.md+++ b/crates/libsy-llm-client/README.md@@ -144,9 +144,9 @@ async fn stream(### Routing an algorithm[`run`] takes a libsy algorithm and a [`ClientRouter`], and returns the final response plus-the trace of decisions the algorithm published. The router resolves each offloaded call to-the client for the target the algorithm selected; `ClientRouter::single` is the-single-provider case:+the trace of decisions the algorithm published. Each offloaded `CallModel` carries an ordered+`models` list. The router resolves and tries those candidates in order; `ClientRouter::single`+is the single-provider case:```rustuse std::sync::Arc;@@ -249,9 +249,12 @@ fn build_multi_format_client(transport failures are retried; streaming body failures are not replayed afterthe response has been returned.-Retries replay the same upstream request. A transport failure can therefore-duplicate a request that the provider processed but did not finish returning,-and the retry budget plus capped `Retry-After` delays determines total latency.+Retries replay the same upstream request to the same model. Each candidate's+`max_retries` budget is exhausted before candidate fallback advances to the next+model. The worst case is `candidates × (max_retries + 1)` upstream requests, and+total latency includes every candidate's capped `Retry-After` backoff. A transport+failure can duplicate a request that the provider processed but did not finish+returning.## Errorsdiff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rsindex e20d1a885..c02475b4b 100644--- a/crates/libsy-llm-client/src/observability.rs+++ b/crates/libsy-llm-client/src/observability.rs@@ -177,7 +177,6 @@ fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),LibsyError::Driver(_) => Cow::Borrowed("driver_error"),LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),- LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),LibsyError::External { .. } => Cow::Borrowed("_OTHER"),}}diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rsindex 1bcf66244..9ef133550 100644--- a/crates/libsy-llm-client/src/run.rs+++ b/crates/libsy-llm-client/src/run.rs@@ -8,8 +8,10 @@//! consumer — it drives the stream with [`switchyard_libsy::drive`], hands each call to a//! [`RoutedLlmClient`], and returns the final response with the trace of decisions.//!-//! libsy owns the stream mechanics; what this module adds is the client call itself and the-//! `libsy.client_call` span around it.+//! libsy owns the stream mechanics; what this module adds is ordered candidate fallback and the+//! `libsy.client_call` span around each candidate. Each candidate exhausts its backend retry+//! budget before fallback advances, so the worst case is `candidates × (max_retries + 1)`+//! upstream attempts plus every candidate's backoff.use std::collections::HashMap;use std::sync::Arc;@@ -17,7 +19,9 @@ use std::time::{Duration, Instant};use parking_lot::Mutex;use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive};-use switchyard_protocol::{Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient};+use switchyard_protocol::{+ Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient, RoutingFallbackReason,+};use crate::observation::{LlmCallObservation, RunObservation, RunObserver};use crate::{metrics, observability};@@ -98,10 +102,60 @@ impl RoutedCallWindows {}}-/// Serve one offloaded call. A failed *model* call is forwarded to the algorithm via-/// `respond`; this errors only when the promise itself could not be fulfilled. `serve` makes-/// the one provider call a routed request performs, so it gets its own `libsy.client_call`-/// span.+/// Serve one offloaded call and fulfill its promise.+///+/// Errors only when the promise itself could not be fulfilled; a call that failed on every+/// candidate is forwarded to the algorithm as an `Err`.+async fn serve(+ clients: ClientRouter,+ call: CallModel,+ observer: Option<RunObserver>,+ // Output parameter because `drive` takes a function that returns a plain `Result<()>`.+ routed_calls: Arc<Mutex<RoutedCallWindows>>,+) -> Result<()> {+ let result = call_first_available(&clients, &call, &observer, &routed_calls).await;+ call.respond(result)+}++/// Try candidates in order until one succeeds or a failure stops fallback.+async fn call_first_available(+ clients: &ClientRouter,+ call: &CallModel,+ observer: &Option<RunObserver>,+ routed_calls: &Arc<Mutex<RoutedCallWindows>>,+) -> Result<Response> {+ for (index, target) in call.models.iter().enumerate() {+ let request = request_for(&call.request, target);+ match call_one(+ clients,+ target,+ request,+ call,+ observer,+ routed_calls,+ index,+ call.models.len(),+ )+ .await+ {+ Ok(response) => return Ok(response),+ Err(error) if index + 1 == call.models.len() => return Err(error),+ Err(error) => match fallback_reason(&error) {+ Some(reason) => tracing::info!(+ from = %target,+ to = %call.models[index + 1],+ reason = reason.as_str(),+ "model call failed; trying next candidate"+ ),+ None => return Err(error),+ },+ }+ }+ Err(LibsyError::NoTargets)+}++/// Call one candidate model and record its observation and span.+#[allow(clippy::too_many_arguments)]#[tracing::instrument(target = "libsy",name = "libsy.client_call",@@ -109,12 +163,14 @@ impl RoutedCallWindows {fields(algorithm = call.algorithm,switchyard.algorithm = call.algorithm,- selected_model = call.selected_model_id(),+ switchyard.candidate = index + 1,+ switchyard.candidate_count = count,+ selected_model = %model_id,otel.kind = "client",- otel.name = %format_args!("chat {}", call.selected_model_id()),+ otel.name = %format_args!("chat {model_id}"),openinference.span.kind = "LLM",gen_ai.operation.name = "chat",- gen_ai.request.model = call.selected_model_id(),+ gen_ai.request.model = %model_id,gen_ai.request.stream = tracing::field::Empty,gen_ai.request.temperature = tracing::field::Empty,gen_ai.request.top_p = tracing::field::Empty,@@ -138,15 +194,20 @@ impl RoutedCallWindows {error = tracing::field::Empty,))]-async fn serve(- clients: ClientRouter,- call: CallModel,- observer: Option<RunObserver>,- // Output parameter because `drive` takes a function that returns a plain `Result<()>`.- routed_calls: Arc<Mutex<RoutedCallWindows>>,-) -> Result<()> {+async fn call_one(+ clients: &ClientRouter,+ model_id: &ModelId,+ request: Request,+ call: &CallModel,+ observer: &Option<RunObserver>,+ routed_calls: &Arc<Mutex<RoutedCallWindows>>,+ // index is for span log+ index: usize,+ // count is for span log+ count: usize,+) -> Result<Response> {let span = tracing::Span::current();- observability::record_gen_ai_request(&span, &call.request.llm_request);+ observability::record_gen_ai_request(&span, &request.llm_request);if let Some(session_id) = call.request.metadata@@ -155,25 +216,27 @@ async fn serve({span.record("gen_ai.conversation.id", session_id);}- let target = ModelId::from(call.selected_model_id());- let request = call.request.clone();- let is_answer_call = call.decision.is_answer_call();+ let is_answer_call = call.is_answer_call;// Resolved before the clock starts: picking the client is Switchyard's work, not// the provider's, so it belongs in the routing overhead.- let client = clients.route(&target);+ let client = clients.route(model_id);let started = Instant::now();let result = match client {Ok(client) => client.call(request).await,Err(error) => Err(error),}- .map_err(|source| LibsyError::client_call(target.clone(), source));+ .map_err(|source| LibsyError::client_call(model_id.clone(), source));let ended = Instant::now();let duration = ended - started;+ let result = result.map(|mut response| {+ response.set_served_model(model_id);+ response+ });let result = observability::observe_client_call(result);if let Some(observer) = observer {observer(RunObservation::LlmCall(LlmCallObservation {- selected_model: target,+ selected_model: model_id.clone(),is_answer_call,is_success: result.is_ok(),duration,@@ -188,7 +251,33 @@ async fn serve(routed_calls.lock().record(started, ended);}- call.respond(result)+ result+}++/// Whether a failed candidate is worth routing around.+fn fallback_reason(error: &LibsyError) -> Option<RoutingFallbackReason> {+ let LibsyError::ClientCall { source, .. } = error else {+ return None;+ };+ match source {+ LlmClientError::ContextWindowExceeded { .. } => Some(RoutingFallbackReason::ContextWindow),+ LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => {+ Some(RoutingFallbackReason::Unavailable)+ }+ LlmClientError::UpstreamHttp { status, .. }+ if matches!(*status, 403 | 408 | 429) || (500..=599).contains(status) =>+ {+ Some(RoutingFallbackReason::Unavailable)+ }+ _ => None,+ }+}++/// Clone a request and stamp the candidate model that should receive it.+fn request_for(request: &Request, target: &ModelId) -> Request {+ let mut request = request.clone();+ request.llm_request.model = Some(target.to_string());+ request}/// Resolves a routed call's selected model to the client that serves it.@@ -256,3 +345,278 @@ impl FromIterator<(ModelId, Arc<dyn RoutedLlmClient>)> for ClientRouter {Self::new(iter.into_iter().collect())}}++#[cfg(test)]+mod tests {+ use std::collections::BTreeMap;++ use super::*;+ use async_trait::async_trait;+ use futures::StreamExt;+ use switchyard_libsy::Driver;+ use switchyard_protocol::{+ LlmResponse, LlmResponseChunk, LlmResponseStreamEvent, completion_text, text_request,+ text_response,+ };+ use wiremock::matchers::method;+ use wiremock::{Mock, MockServer, ResponseTemplate};++ use crate::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};++ struct CandidateAlgorithm {+ models: Vec<ModelId>,+ }++ #[async_trait]+ impl Algorithm for CandidateAlgorithm {+ fn name(&self) -> &str {+ "candidate_test"+ }++ async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {+ driver.call_model(request, self.models.clone(), true).await+ }+ }++ #[derive(Clone, Copy)]+ enum FirstOutcome {+ ContextWindow,+ Unauthorized,+ StreamSuccess,+ MidStreamError,+ }++ struct CandidateClient {+ calls: Mutex<Vec<ModelId>>,+ first: FirstOutcome,+ }++ #[async_trait]+ impl RoutedLlmClient for CandidateClient {+ async fn call(&self, request: Request) -> std::result::Result<Response, LlmClientError> {+ let model = request.model_id().unwrap_or_default();+ self.calls.lock().push(model.clone());+ if model == "weak" {+ return match self.first {+ FirstOutcome::ContextWindow => Err(LlmClientError::ContextWindowExceeded {+ model,+ message: "too long".to_string(),+ }),+ FirstOutcome::Unauthorized => Err(LlmClientError::UpstreamHttp {+ status: 401,+ body: "unauthorized".to_string(),+ }),+ FirstOutcome::StreamSuccess => Ok(stream_response(vec![+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "streamed".to_string(),+ },+ LlmResponseChunk::MessageStop {+ reason: Some("stop".to_string()),+ },+ ])),+ FirstOutcome::MidStreamError => Ok(stream_response(vec![+ LlmResponseChunk::TextDelta {+ index: 0,+ text: "partial".to_string(),+ },+ LlmResponseChunk::StreamError {+ message: "stream failed".to_string(),+ },+ ])),+ };+ }+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), model)),+ metadata: None,+ })+ }+ }++ fn stream_response(chunks: Vec<LlmResponseChunk>) -> Response {+ Response {+ llm_response: LlmResponse::Stream(+ futures::stream::iter(+ chunks+ .into_iter()+ .map(|chunk| Ok(LlmResponseStreamEvent::from(chunk))),+ )+ .boxed(),+ ),+ metadata: None,+ }+ }++ fn request() -> Request {+ Request {+ llm_request: text_request(Some("auto".to_string()), "hello".to_string()),+ raw_request: None,+ metadata: None,+ }+ }++ async fn run_candidates(+ first: FirstOutcome,+ ) -> (Arc<CandidateClient>, Result<(Vec<Decision>, Response)>) {+ let client = Arc::new(CandidateClient {+ calls: Mutex::new(Vec::new()),+ first,+ });+ let algorithm = Arc::new(CandidateAlgorithm {+ models: vec!["weak".into(), "strong".into()],+ });+ let result = run(+ algorithm,+ ClientRouter::single(client.clone()),+ request(),+ None,+ )+ .await;+ (client, result)+ }++ #[test]+ fn fallback_only_accepts_context_and_unavailable_failures() {+ let error = |source| LibsyError::client_call("target", source);+ assert_eq!(+ fallback_reason(&error(LlmClientError::ContextWindowExceeded {+ model: "target".into(),+ message: "too long".to_string(),+ })),+ Some(RoutingFallbackReason::ContextWindow)+ );+ for status in [403, 408, 429, 500, 599] {+ assert_eq!(+ fallback_reason(&error(LlmClientError::UpstreamHttp {+ status,+ body: "failed".to_string(),+ })),+ Some(RoutingFallbackReason::Unavailable)+ );+ }+ for status in [400, 401, 404, 409, 499, 600] {+ assert_eq!(+ fallback_reason(&error(LlmClientError::UpstreamHttp {+ status,+ body: "failed".to_string(),+ })),+ None+ );+ }+ }++ #[tokio::test]+ async fn candidate_failures_follow_the_fallback_policy() -> Result<()> {+ // Context overflow is retryable across candidates.+ let (client, result) = run_candidates(FirstOutcome::ContextWindow).await;+ let (_, response) = result?;+ assert_eq!(+ &*client.calls.lock(),+ &[ModelId::from("weak"), "strong".into()]+ );+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(|response| response.model.as_deref()),+ Some(Some("strong"))+ );+ assert_eq!(response.served_model().map(ModelId::as_str), Some("strong"));++ // Authentication failure is not retryable, so the second candidate is untouched.+ let (client, result) = run_candidates(FirstOutcome::Unauthorized).await;+ assert!(matches!(+ result,+ Err(LibsyError::ClientCall {+ source: LlmClientError::UpstreamHttp { status: 401, .. },+ ..+ })+ ));+ assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);+ Ok(())+ }++ #[tokio::test]+ async fn retry_budget_is_exhausted_before_falling_through() -> Result<()> {+ let server = MockServer::start().await;+ let calls = Arc::new(Mutex::new(Vec::new()));+ let observed_calls = Arc::clone(&calls);+ Mock::given(method("POST"))+ .respond_with(move |request: &wiremock::Request| {+ let body: serde_json::Value =+ serde_json::from_slice(&request.body).unwrap_or(serde_json::Value::Null);+ let model = body["model"].as_str().unwrap_or_default().to_string();+ observed_calls.lock().push(model.clone());+ if model == "weak" {+ ResponseTemplate::new(503)+ .insert_header("retry-after", "0")+ .set_body_string("unavailable")+ } else {+ ResponseTemplate::new(200).set_body_json(serde_json::json!({+ "id": "answer",+ "model": "strong",+ "choices": [{+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop"+ }],+ "usage": {}+ }))+ }+ })+ .mount(&server)+ .await;++ let backend = || {+ Backend::OpenAiChat(HttpBackendConfig {+ base_url: format!("{}/v1", server.uri()),+ api_key: None,+ forward_auth: false,+ extra_headers: BTreeMap::new(),+ extra_body: BTreeMap::new(),+ max_retries: 2,+ })+ };+ let client = Arc::new(+ TranslatingLlmClient::new(&[+ ModelConfig::new("weak", backend(), None),+ ModelConfig::new("strong", backend(), None),+ ])+ .map_err(|error| LibsyError::external("building test client", error))?,+ );+ let algorithm = Arc::new(CandidateAlgorithm {+ models: vec!["weak".into(), "strong".into()],+ });+ run(algorithm, ClientRouter::single(client), request(), None).await?;++ assert_eq!(&*calls.lock(), &["weak", "weak", "weak", "strong"]);+ Ok(())+ }++ #[tokio::test]+ async fn streams_are_outside_the_candidate_fallback_boundary() -> Result<()> {+ // Receiving a stream handle is a successful call and ends candidate selection.+ let (client, result) = run_candidates(FirstOutcome::StreamSuccess).await;+ let (_, response) = result?;+ assert_eq!(response.served_model().map(ModelId::as_str), Some("weak"));+ let aggregate = response+ .llm_response+ .into_agg()+ .await+ .map_err(|source| LibsyError::client_call("weak", source))?;+ assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);+ assert_eq!(completion_text(&aggregate), "streamed");++ // A later in-stream failure surfaces during aggregation without trying another model.+ let (client, result) = run_candidates(FirstOutcome::MidStreamError).await;+ let (_, response) = result?;+ let error = response+ .llm_response+ .into_agg()+ .await+ .expect_err("the stream should fail during aggregation");+ assert!(error.to_string().contains("stream failed"));+ assert_eq!(&*client.calls.lock(), &[ModelId::from("weak")]);+ Ok(())+ }+}diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex aefae5bc2..dd5afa35e 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -444,7 +444,7 @@ impl Algorithm for SingleCallAlgo {tracing::info!("picked '{target}'");let decision = Decision::new(target.clone(), true);driver.decide(decision.clone()).await?;- driver.call_model(request, decision).await+ driver.call_model(request, vec![target.into()], true).await}}diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 895c69f82..08072689c 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -33,9 +33,10 @@ tokio = { version = "1", features = ["macros", "rt"] }A target is a bare model id naming a routing destination. An [`Algorithm`] selects targets andrecords [`Decision`](switchyard_protocol::Decision)s, offloading every model callto its caller: [`Algorithm::run_stream`] yields a [`Step`] stream whose-[`Step::CallModel`] items the host serves over its own transport. libsy makes no-network calls itself — `switchyard-llm-client`'s `run` is a ready-made consumer-that drives the stream and performs the calls over HTTP.+[`Step::CallModel`] items the host serves over its own transport. Each call carries+an ordered, non-empty list of candidate models; the host tries them until one+answers. libsy makes no network calls itself — `switchyard-llm-client`'s `run` is+a ready-made consumer that drives the stream and performs the calls over HTTP.The provider-neutral [`Request`], [`Response`], [`Usage`], and [`LlmResponse`]contracts come from `switchyard-protocol`.diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 314f3440f..be385a70e 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -13,11 +13,9 @@//! private state value across turns with the same session ID. Requests without a session ID use//! unretained per-run state.//!-//! Every composition retains one thing regardless: a target that overflows its context window is-//! remembered for the rest of its session and skipped on later turns.-//! An unavailable target is skipped only for the current request.+//! The selected target is offered first, followed by every other configured target. The consumer+//! may fall through that ordered candidate list when a model call fails.-use std::collections::HashSet;use std::{collections::HashMap,sync::{Arc, Once, Weak},@@ -28,11 +26,11 @@ use async_trait::async_trait;use parking_lot::Mutex;use tokio::sync::Mutex as AsyncMutex;-use crate::core::algorithm::{self, Algorithm, Driver, RoutingIdentity, SessionEvictions};+use crate::core::algorithm::{self, Algorithm, Driver};use crate::core::classifier::{Classification, Classifier, Score};use crate::core::processor::{Event, Processor};use crate::{LibsyError, Result};-use switchyard_protocol::{Decision, ModelId, Request, Response, RoutingFallbackReason};+use switchyard_protocol::{Decision, ModelId, Request, Response};struct SessionState<S> {state: Arc<AsyncMutex<S>>,@@ -99,7 +97,6 @@ pub struct FallThrough<S = ()> {targets: Vec<ModelId>,session_states: Option<Arc<SessionStates<S>>>,cleanup_started: Once,- session_evictions: SessionEvictions,}impl FallThrough<()> {@@ -113,7 +110,6 @@ impl FallThrough<()> {targets,session_states: None,cleanup_started: Once::new(),- session_evictions: SessionEvictions::default(),}}}@@ -132,7 +128,6 @@ wheretargets,session_states: Some(Arc::new(Mutex::new(HashMap::new()))),cleanup_started: Once::new(),- session_evictions: SessionEvictions::default(),}}@@ -190,56 +185,29 @@ where// The request is threaded mutably through the whole fold: any component may rewrite// it, later components see the rewrite, and the final value reaches the model.let mut request = request;- // Targets this turn must not route to. Scratch state for one run: seeded from the- // session's overflow history, then grown by each route-level failure below.- let mut excluded = HashSet::new();- // Processors may rewrite the request; overflow history stays with its inbound identity.- let identity = RoutingIdentity::from_request(&request);- algorithm::exclude_evicted(- &mut excluded,- &self.targets,- &self.session_evictions,- identity.as_ref(),- );let session_state = self.session_state(&request);- let (target, decision, served, deciding) = match session_state {+ let (target, served) = match session_state {Some(state) => {let mut state = state.lock().await;- self.route(&mut state, &excluded, &driver, &mut request)- .await?+ self.route(&mut state, &driver, &mut request).await?}None => {let mut state = S::default();- self.route(&mut state, &excluded, &driver, &mut request)- .await?+ self.route(&mut state, &driver, &mut request).await?}};// A classifier that already called a model — because deciding required one, and that// call also answers the turn — hands its response back here, so the turn is not paid- // for twice. There is no outbound call left to overflow, so the fallback is skipped.+ // for twice.// Nothing reads it on the way out: streamed or buffered, it reaches the caller// untouched.match served {Some(response) => Ok(response),None => {- algorithm::call_model_with_fallback(- &mut excluded,- &driver,- &self.targets,- target,- decision,- request,- identity.as_ref(),- &self.session_evictions,- |request, target| {- for classifier in &self.classifiers {- classifier.target_unavailable(request, target);- }- },- |from, to, reason| self.fallback_decision(deciding.as_ref(), from, to, reason),- )- .await+ driver+ .call_model(request, self.candidates(&target), true)+ .await}}}@@ -249,30 +217,18 @@ whereif let Some(states) = &self.session_states {states.lock().remove(session);}- self.session_evictions.remove_session(session);}- /// The decision published when a route-level failure selects a different target.- fn fallback_decision(- &self,- deciding: &dyn Classifier<S>,- from: &ModelId,- to: &ModelId,- reason: RoutingFallbackReason,- ) -> Decision {- let failure = match reason {- RoutingFallbackReason::ContextWindow => "exceeded its context window",- RoutingFallbackReason::Unavailable => "was unavailable",- };- let message = with_routing_tier(- format!(- "{from} {failure}; fell back to {to} (fallback reason: {})",- reason.as_str(),- ),- deciding.routing_tier(to),- );- tracing::info!("{message}");- Decision::new(to.clone(), true)+ /// The selected target first, then every other configured target as a fallback candidate.+ fn candidates(&self, target: &ModelId) -> Vec<ModelId> {+ std::iter::once(target.clone())+ .chain(+ self.targets+ .iter()+ .filter(|candidate| *candidate != target)+ .cloned(),+ )+ .collect()}/// Returns this request's retained state without holding the registry lock.@@ -292,10 +248,9 @@ whereasync fn route(&self,state: &mut S,- excluded: &HashSet<ModelId>,driver: &Driver,request: &mut Request,- ) -> Result<(ModelId, Decision, Option<Response>, Arc<dyn Classifier<S>>)> {+ ) -> Result<(ModelId, Option<Response>)> {// 1. Processor chain accumulates request-side facts into the composition's state.for processor in &self.processors {processor.process(state, Event::Request(request)).await?;@@ -319,17 +274,10 @@ where});};- // 3. Resolve the target, log the choice, and publish the decision. When an excluded- // target sends the request elsewhere, the log describes where it actually went.- let target = algorithm::select_eligible_model(&self.targets, &score.target, excluded)?;- let message = if target == score.target {- (self.decision_reason)(&self.name, &score)- } else {- format!(- "{} exceeded its context window; fell back to {}",- score.target, target- )- };+ // 3. Resolve the target, log the choice, and publish the decision.+ algorithm::ensure_model_is_target(&self.targets, &score.target)?;+ let target = score.target.clone();+ let message = (self.decision_reason)(&self.name, &score);let message = with_routing_tier(message, deciding.routing_tier(&target));tracing::info!("{message}");let decision: Decision = Decision::new(target.clone(), true);@@ -345,7 +293,7 @@ whereprocessor.process(state, event).await?;}- Ok((target, decision, served, deciding))+ Ok((target, served))}}@@ -417,12 +365,10 @@ mod tests {use super::*;use crate::algorithms::util::prompts;use crate::core::classifier::Classification;- use crate::{AffinityRouter, SystemPromptProcessor, TargetPrompts};+ use crate::{SystemPromptProcessor, TargetPrompts};use crate::core::testing::{Serve, echo, reply, test_drive};- use switchyard_protocol::{- LlmClientError, LlmRequest, Message, Metadata, Role, completion_text, text_request,- };+ use switchyard_protocol::{LlmRequest, Message, Metadata, Role, completion_text, text_request};#[derive(Debug, thiserror::Error)]#[error("{0}")]@@ -437,11 +383,11 @@ mod tests {/// Echoes the routed model name, capturing the request it was handed so a test can/// assert on what actually reached the model.fn capturing(into: Arc<Mutex<Option<Request>>>) -> impl Serve {- move |decision: Decision, request: Request| {+ move |target: ModelId, request: Request| {let into = Arc::clone(&into);async move {*into.lock() = Some(request);- Ok(reply(decision.selected_model_id()))+ Ok(reply(target))}}}@@ -465,11 +411,11 @@ mod tests {impl PromptRecorder {fn serve(self: &Arc<Self>) -> impl Serve {let recorder = Arc::clone(self);- move |decision: Decision, request: Request| {+ move |target: ModelId, request: Request| {let recorder = Arc::clone(&recorder);async move {*recorder.0.lock() = Some(RecordedCall {- target: decision.selected_model_id().to_string(),+ target: target.to_string(),messages: request.llm_request.messages@@ -483,7 +429,7 @@ mod tests {.filter_map(|block| block.content.iter().find_map(text_of)).collect(),});- Ok(reply(decision.selected_model_id()))+ Ok(reply(target))}}}@@ -622,55 +568,6 @@ mod tests {run_turn(&Arc::new(router), serve).await}- /// Discards the recorded calls; the overflow tests that don't assert on them.- fn overflows(targets: &'static [&'static str]) -> impl Serve {- overflowing(targets, Arc::new(Mutex::new(Vec::new())))- }-- /// Rejects the named `overflowing` targets with a context-window error and echoes for- /// the rest, so the retry path can be driven. Every call is recorded in `calls`.- fn overflowing(- overflowing: &'static [&'static str],- calls: Arc<Mutex<Vec<String>>>,- ) -> impl Serve {- move |decision: Decision, _request: Request| {- let calls = Arc::clone(&calls);- async move {- let model = decision.selected_model_id().clone();- calls.lock().push(model.to_string());- if overflowing.contains(&model.as_str()) {- return Err(LlmClientError::ContextWindowExceeded {- model,- message: "prompt is too long".to_string(),- });- }- Ok(reply(model))- }- }- }-- /// Rejects the named `unavailable` targets with a 503 and echoes for the rest.- /// Every call is recorded in `calls`.- fn unavailable(- unavailable: &'static [&'static str],- calls: Arc<Mutex<Vec<String>>>,- ) -> impl Serve {- move |decision: Decision, _request: Request| {- let calls = Arc::clone(&calls);- async move {- let model = decision.selected_model_id().to_string();- calls.lock().push(model.clone());- if unavailable.contains(&model.as_str()) {- return Err(LlmClientError::UpstreamHttp {- status: 503,- body: "unavailable".to_string(),- });- }- Ok(reply(model))- }- }- }-// --- tests -------------------------------------------------------------------------#[tokio::test]@@ -760,187 +657,23 @@ mod tests {}#[tokio::test]- async fn a_target_that_overflowed_is_skipped_for_the_rest_of_the_session() -> Result<()> {- let calls = Arc::new(Mutex::new(Vec::new()));- let router = Arc::new(- FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)])),- );- for _ in 0..3 {- let serve = overflowing(&["weak"], calls.clone());- assert_eq!(run_turn(&router, serve).await?.0, "strong");- }- assert_eq!(calls.lock().iter().filter(|m| *m == "weak").count(), 1);- Ok(())- }-- #[tokio::test]- async fn a_different_session_starts_with_an_empty_eviction_set() -> Result<()> {- let calls = Arc::new(Mutex::new(Vec::new()));- let router = Arc::new(- FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)])),- );- run_turn(&router, overflowing(&["weak"], calls.clone())).await?;- let mut other = request();- other.metadata = Some(Metadata {- session_id: Some("session-2".to_string()),- ..Metadata::default()- });- run_request(&router, other, overflowing(&["weak"], calls.clone())).await?;- assert_eq!(calls.lock().iter().filter(|m| *m == "weak").count(), 2);- Ok(())- }-- #[tokio::test]- async fn second_turn_after_full_exhaustion_still_reaches_upstream() -> Result<()> {- let calls = Arc::new(Mutex::new(Vec::new()));- let router = Arc::new(- FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)])),- );- let first = run_turn(&router, overflowing(&["weak", "strong"], calls.clone())).await;- assert!(first.is_err());- calls.lock().clear();- match run_turn(&router, overflowing(&["weak", "strong"], calls.clone())).await {- Err(LibsyError::ClientCall { .. }) => {}- Err(other) => panic!("turn 2 gave {other:?}, calls={:?}", calls.lock()),- Ok(_) => panic!("expected an error"),- }- Ok(())- }+ async fn selected_target_leads_the_ordered_candidate_list() -> Result<()> {+ use futures::StreamExt;- #[tokio::test]- async fn an_overflowing_target_is_retried_on_one_that_fits() -> Result<()> {- let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)]));- let (model, _) = run_with(router, overflows(&["weak"])).await?;- assert_eq!(model, "strong");- Ok(())- }-- #[tokio::test]- async fn unavailable_target_clears_matching_affinity_before_the_next_turn() -> Result<()> {- let calls = Arc::new(Mutex::new(Vec::new()));- let affinity = Arc::new(AffinityRouter::new());let router = Arc::new(- FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_processor(affinity.clone())- .with_classifier(affinity)- .with_classifier(Arc::new(DefaultTarget::new("weak"))),+ FallThrough::<()>::new(target_set(&["weak", "mid", "strong"]))+ .with_classifier(fixed(vec![score("mid", 0.9)])),);- for _ in 0..2 {- let (model, trace) = run_turn(&router, unavailable(&["weak"], calls.clone())).await?;- assert_eq!(model, "strong");- assert_eq!(- trace- .last()- .map(|decision| decision.selected_model_id().as_str()),- Some("strong")- );- }- assert_eq!(&*calls.lock(), &["weak", "strong", "weak", "strong"]);- Ok(())- }-- #[tokio::test]- async fn fallback_decision_preserves_answer_call_semantics() -> Result<()> {- struct TieredClassifier;-- #[async_trait]- impl Classifier for TieredClassifier {- async fn score(- &self,- _state: &mut (),- _request: &mut Request,- _driver: Option<&Driver>,- ) -> Result<(Classification, Option<Response>)> {- Ok((Classification::Scores(vec![score("weak", 1.0)]), None))+ let stream = router.run_stream(request());+ tokio::pin!(stream);+ while let Some(step) = stream.next().await {+ if let crate::Step::CallModel(call) = step? {+ assert_eq!(call.models, target_set(&["mid", "weak", "strong"]));+ assert_eq!(call.request.llm_request.model.as_deref(), Some("mid"));+ return Ok(());}}-- let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(Arc::new(TieredClassifier));- let (_, trace) = run_with(router, unavailable(&["weak"], Arc::default())).await?;-- assert_eq!(trace.len(), 2);- assert_eq!(trace[0].selected_model_id(), "weak");- assert!(trace[0].is_answer_call());-- let fallback = &trace[1];- assert_eq!(fallback.selected_model_id(), "strong");- assert!(fallback.is_answer_call());- Ok(())- }-- #[tokio::test]- async fn overflowing_targets_are_retried_until_one_fits() -> Result<()> {- let router = FallThrough::<()>::new(target_set(&["weak", "mid", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)]));- let (model, _) = run_with(router, overflows(&["weak", "mid"])).await?;- assert_eq!(model, "strong");- Ok(())- }-- #[tokio::test]- async fn exhausting_every_target_surfaces_the_client_overflow() -> Result<()> {- // Only the client error maps to a 400 upstream, so it must survive exhaustion.- let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)]));- match run_with(router, overflows(&["weak", "strong"])).await {- Ok(_) => panic!("expected an overflow error, got a response"),- Err(LibsyError::ClientCall {- source: LlmClientError::ContextWindowExceeded { .. },- ..- }) => Ok(()),- Err(other) => panic!("expected ContextWindowExceeded, got {other:?}"),- }- }-- #[tokio::test]- async fn a_retried_request_runs_the_processors_once() -> Result<()> {- // Routing runs before the call loop, so an overflow must not replay processors.- struct CountingProcessor(Arc<Mutex<Vec<&'static str>>>);-- #[async_trait]- impl Processor for CountingProcessor {- async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {- let kind = match event {- Event::Request(_) => "request",- Event::Decision { .. } => "decision",- _ => "other",- };- self.0.lock().push(kind);- Ok(())- }- }-- let seen = Arc::new(Mutex::new(Vec::new()));- let router = FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)]))- .with_processor(Arc::new(CountingProcessor(seen.clone())));- let (model, _) = run_with(router, overflows(&["weak"])).await?;- assert_eq!(model, "strong");- assert_eq!(seen.lock().iter().filter(|e| **e == "request").count(), 1);- assert_eq!(seen.lock().iter().filter(|e| **e == "decision").count(), 1);- Ok(())- }-- #[tokio::test]- async fn a_target_barred_by_overflow_history_reports_the_fallback() -> Result<()> {- // Headers and usage metrics read the decision, so it must describe the real call.- let router = Arc::new(- FallThrough::<()>::new(target_set(&["weak", "strong"]))- .with_classifier(fixed(vec![score("weak", 0.9)])),- );- // The first turn teaches the session that "weak" overflows; the second is barred- // from it before calling, so routing redirects and must say where it went.- run_turn(&router, overflows(&["weak"])).await?;- let (text, trace) = run_turn(&router, overflows(&["weak"])).await?;-- assert_eq!(text, "strong");- assert_eq!(trace[0].selected_model_id(), "strong");- Ok(())+ Err(test_error("expected a CallModel step"))}#[tokio::test]diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 7fec82dc3..7d8b2fb66 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -9,7 +9,7 @@ use std::sync::Arc;use async_trait::async_trait;use serde::{Deserialize, Deserializer};use serde_json::Value;-use switchyard_protocol::{ContentBlock, Decision, Message, ModelId, Role};+use switchyard_protocol::{ContentBlock, Message, ModelId, Role};use super::fall_through::{DefaultTarget, FallThrough};use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;@@ -515,15 +515,14 @@ impl Classifier<State> for EscalationClassifier {// Call efficient model and buffer the response so the judge can read it.//- // If the efficient model exceeds its context window, fall through to capable: returning- // `(decisive(capable), None)` tells FallThrough::execute to call- // call_model_with_fallback with the capable target instead of surfacing the error.+ // If the efficient model exceeds its context window, fall through to capable. This call+ // deliberately has one candidate so the classifier sees the efficient model's error.tracing::info!(target = %self.efficient,"escalation classifier selected efficient tier");let efficient_response = match driver- .call_model(request.clone(), Decision::new(self.efficient.clone(), true))+ .call_model(request.clone(), vec![self.efficient.clone()], true).await{Ok(r) => r,@@ -1020,15 +1019,15 @@ mod tests {fn serve(self: &Arc<Self>) -> impl Serve {let recorder = Arc::clone(self);- move |decision: Decision, request: Request| {+ move |model: ModelId, request: Request| {let recorder = Arc::clone(&recorder);async move {- let model = decision.selected_model_id().to_string();+ let model = model.to_string();recorder.calls.lock().push(model.clone());recorder.call_roles.lock()- .push((model.clone(), decision.is_answer_call()));+ .push((model.clone(), model != "judge"));let completion = if model == "judge" {recorder.judge_max_output_tokens@@ -1064,8 +1063,8 @@ mod tests {/// The judge times out; every other target answers normally.fn unreachable_judge() -> impl Serve {- |decision: Decision, request: Request| async move {- let model = decision.selected_model_id().to_string();+ |model: ModelId, request: Request| async move {+ let model = model.to_string();if model == "judge" {return Err(LlmClientError::Timeout {source: Box::new(std::io::Error::other("judge unreachable")),@@ -1739,8 +1738,6 @@ mod tests {use std::collections::VecDeque;- use switchyard_protocol::Decision;-/// A queue of replies, drained in order.struct Queue(Mutex<VecDeque<String>>);@@ -1762,8 +1759,8 @@ mod tests {/// Serves the judge target from `judge` and every other target from `model`, each with/// its next queued reply.fn queued(model: Arc<Queue>, judge: Arc<Queue>) -> impl Serve {- move |decision: Decision, request: Request| {- let queue = if decision.selected_model_id() == "judge" {+ move |target: ModelId, request: Request| {+ let queue = if target == "judge" {Arc::clone(&judge)} else {Arc::clone(&model)@@ -1908,10 +1905,10 @@ mod tests {let router = escalation_router()?;// Efficient overflows, capable answers, and the judge must never be called.- let serve = |decision: Decision, _request: Request| async move {- match decision.selected_model_id().as_str() {+ let serve = |target: ModelId, _request: Request| async move {+ match target.as_str() {"efficient" => Err(LlmClientError::ContextWindowExceeded {- model: decision.selected_model_id().clone(),+ model: target,message: "prompt is too long".to_string(),}),"judge" => panic!("the judge must not be consulted when efficient overflows"),@@ -1940,10 +1937,10 @@ mod tests {let calls = Arc::new(Mutex::new(Vec::new()));let serve = {let calls = Arc::clone(&calls);- move |decision: Decision, _request: Request| {+ move |model: ModelId, _request: Request| {let calls = Arc::clone(&calls);async move {- let model = decision.selected_model_id().to_string();+ let model = model.to_string();calls.lock().push(model.clone());match model.as_str() {"efficient" => Ok(streamed_then_error(LlmClientError::Transport {@@ -1975,8 +1972,8 @@ mod tests {#[tokio::test]async fn escalation_classifier_preserves_non_transport_stream_errors() -> Result<()> {let router = escalation_router()?;- let serve = |decision: Decision, _request: Request| async move {- match decision.selected_model_id().as_str() {+ let serve = |target: ModelId, _request: Request| async move {+ match target.as_str() {"efficient" => Ok(streamed_then_error(LlmClientError::InvalidResponse {source: Box::new(std::io::Error::other("invalid stream event")),})),diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex f123c1d4f..3de5ccc9a 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -35,7 +35,9 @@ impl Algorithm for Passthrough {tracing::info!(target = %self.target, "passthrough selected target");let decision: Decision = Decision::new(self.target.clone(), true);driver.decide(decision.clone()).await?;- driver.call_model(request, decision).await+ driver+ .call_model(request, vec![self.target.clone()], true)+ .await}}diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex 948793586..f751f7939 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -219,7 +219,7 @@ mod tests {use crate::core::classifier::Score;use crate::core::state::StateValue;use crate::core::testing::{Serve, reply, test_drive};- use switchyard_protocol::{Decision, Metadata, Response};+ use switchyard_protocol::{Metadata, Response};/// A classifier that always picks `target`, standing in for a cascade member.struct Fixed(&'static str);@@ -358,10 +358,10 @@ mod tests {/// back so the fallback classifier has an answer without a real model.fn serve(self: &Arc<Self>) -> impl Serve {let recorder = Arc::clone(self);- move |decision: Decision, request: Request| {+ move |target: ModelId, request: Request| {let recorder = Arc::clone(&recorder);async move {- let target = decision.selected_model_id().to_string();+ let target = target.to_string();recorder.calls.lock().push(Call {target: target.clone(),messages: request@@ -370,7 +370,7 @@ mod tests {.iter().filter_map(|message| message.text_content("|")).collect(),- is_answer_call: decision.is_answer_call(),+ is_answer_call: target != JUDGE,});let completion = if target == JUDGE {let p_solve = *recorder.judge_p_solve.lock();diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rsindex d372fdef0..b95f91f15 100644--- a/crates/libsy/src/algorithms/util/affinity.rs+++ b/crates/libsy/src/algorithms/util/affinity.rs@@ -179,19 +179,6 @@ impl<S> Classifier<S> for AffinityRouterwhereS: Send + 'static,{- fn target_unavailable(&self, request: &Request, target: &ModelId) {- let Some(key) = self.affinity_key(request) else {- return;- };- let mut assignments = self.assignments.lock();- if assignments- .get(&key)- .is_some_and(|assigned| assigned == target)- {- assignments.remove(&key);- }- }-async fn score(&self,_state: &mut S,diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 29d7a49c5..4d5cfb564 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -22,7 +22,7 @@ use crate::core::algorithm::Driver;use crate::core::classifier::{Classification, Classifier};use crate::core::state::State;use crate::{LibsyError, Result};-use switchyard_protocol::{Decision, LlmClientError, Request, Response};+use switchyard_protocol::{LlmClientError, Request, Response};/// Builds the classifier-specific message view presented to a structured judge.pub(crate) trait ClassifierInput: Send + Sync {@@ -240,7 +240,8 @@ wherelet response = driver.call_model(self.judge.build_request(state, request),- Decision::new(self.target.to_string(), false),+ vec![self.target.clone()],+ false,).await.inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error)))diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex d75163f8c..f46b4815e 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -5,18 +5,10 @@//! algorithm implements, and the offload channel it uses to make model calls and//! publish [`Decision`]s.-use std::{- collections::{HashMap, HashSet},- future::Future,- panic::AssertUnwindSafe,- pin::Pin,- sync::Arc,- time::Instant,-};+use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::Instant};use async_trait::async_trait;use futures::{FutureExt, Stream, StreamExt};-use parking_lot::Mutex;use tokio::sync::{mpsc, oneshot};use tokio_stream::wrappers::ReceiverStream;use tracing::Instrument;@@ -28,9 +20,7 @@ use tracing::Instrument;/// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and/// [`switchyard_protocol::LlmResponse`] carries either a live/// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.-use switchyard_protocol::{- Decision, LlmClientError, ModelId, Request, Response, RoutingFallbackReason,-};+use switchyard_protocol::{Decision, ModelId, Request, Response};use crate::{DriverError, LibsyError, Result, observability};@@ -47,32 +37,23 @@ pub type StepStream = Pin<Box<dyn Stream<Item = Result<Step>> + Send>>;/// you. A host that only wants the routing outcome can take the contents with/// [`into_parts`](Self::into_parts) and never respond; dropping the stream ends the run.///-/// The selected model is available both from-/// [`decision.selected_model_id()`](Decision::selected_model_id) and from-/// `request.llm_request.model`. [`Driver::call_model`] stamps the decision's model onto the-/// request before publishing the call, so every consumer receives a request ready for the-/// selected target.+/// [`Driver::call_model`] stamps the first candidate model onto the request before publishing+/// the call. A consumer that falls through to a later candidate must re-stamp it.pub struct CallModel {/// The name of the algorithm that produced this call, so a host instrumenting the/// calls it serves can attribute its own spans to the algorithm behind them.pub algorithm: String,- /// The request to serve; its `model` is the selected model identified by `decision`.+ /// The request to serve; its `model` is stamped with the first candidate.pub request: Request,- /// The routing decision behind this call; `selected_model_id()` identifies the model to use.- pub decision: Decision,+ /// Candidate models, tried in order until one answers. Never empty.+ pub models: Vec<ModelId>,+ /// True for an answer-generating call, false for classifier and judge calls.+ pub is_answer_call: bool,// How to send the response back to the algorithmreply: oneshot::Sender<Result<Response>>,}impl CallModel {- /// The selected model stamped onto this call's request by [`Driver::call_model`].- pub fn selected_model_id(&self) -> &str {- let model_id = self.request.llm_request.model.as_deref();- // Driver stamps the model before constructing CallModel.- debug_assert!(model_id.is_some());- model_id.unwrap_or_default()- }-/// Fulfill the promise with the caller's model-call result. Pass `Err(..)` to/// propagate a failed model call back to the algorithm. Consumes the promise: it/// can only be fulfilled once.@@ -85,17 +66,17 @@ impl CallModel {/// Take the call's contents without answering it, dropping the promise — the routing/// outcome plus the request as the algorithm would have sent it, after any rewriting.///- /// Should only be called if `decision.is_answer_call` is true as that is the final call.+ /// Should only be called if `is_answer_call` is true as that is the final call./// The algorithm's [`Driver::call_model`] will fail with [`DriverError::Abandoned`] and/// the run ends there. Taking a call the algorithm does not depend on (a judge or/// classifier call) may instead let it fail open and complete with degraded routing.////// An abandoned run is not recorded as a failed one. Dropping a [`CallModel`] without/// calling this still yields [`DriverError::ResponseDropped`], which is.- pub fn into_parts(self) -> (Request, Decision) {+ pub fn into_parts(self) -> (Request, Vec<ModelId>) {let Self {request,- decision,+ models,reply,..} = self;@@ -103,7 +84,7 @@ impl CallModel {// telemetry can tell an abandoned run from a failed one. The receiver is already// gone if the algorithm stopped waiting, which is fine.let _ = reply.send(Err(DriverError::Abandoned.into()));- (request, decision)+ (request, models)}}@@ -134,8 +115,9 @@ impl Driver {)}- /// Offload a model call: publish it as a [`Step::CallModel`] and await the consumer's- /// [`Response`]. Errors if the stream is closed or the call failed.+ /// Publish a model call and await the consumer's response.+ ///+ /// Errors if the stream is closed or the call failed./// The await is wrapped in a `libsy.llm_call` span measuring *fulfillment* as/// the algorithm observes it (host queueing/serving included; a streamed/// response resolves when its stream handle arrives); latency, outcome, and@@ -147,7 +129,7 @@ impl Driver {skip_all,fields(algorithm = self.algorithm,- selected_model = %decision.selected_model_id(),+ selected_model = %models.first().map(ModelId::as_str).unwrap_or("NoTargets"),openinference.span.kind = "CHAIN",outcome = tracing::field::Empty,error = tracing::field::Empty,@@ -157,16 +139,23 @@ impl Driver {reasoning_tokens = tracing::field::Empty,))]- pub async fn call_model(&self, mut request: Request, decision: Decision) -> Result<Response> {- let selected_model_id = decision.selected_model_id().to_string();- request.llm_request.model = Some(selected_model_id.clone());- let is_answer_call = decision.is_answer_call();+ pub async fn call_model(+ &self,+ mut request: Request,+ models: Vec<ModelId>,+ is_answer_call: bool,+ ) -> Result<Response> {+ let Some(selected_model_id) = models.first().cloned() else {+ return Err(LibsyError::NoTargets);+ };+ request.llm_request.model = Some(selected_model_id.to_string());let started = Instant::now();let (reply, response) = oneshot::channel::<Result<Response>>();let call = CallModel {algorithm: self.algorithm.clone(),request,- decision,+ models,+ is_answer_call,reply,};let result = async {@@ -182,7 +171,7 @@ impl Driver {let elapsed = started.elapsed();observability::record_llm_call(&self.algorithm,- &selected_model_id,+ selected_model_id.as_str(),is_answer_call,elapsed,&result,@@ -313,27 +302,8 @@ pub(crate) fn ensure_model_is_target(targets: &[ModelId], name: &ModelId) -> Res})}-/// `name` itself, or the first target not in `excluded` when `name` has been barred.-/// Errors if `name` is unknown, or if every target is excluded.-pub(crate) fn select_eligible_model(- targets: &[ModelId],- name: &ModelId,- excluded: &HashSet<ModelId>,-) -> Result<ModelId> {- ensure_model_is_target(targets, name)?;- if !excluded.contains(name) {- return Ok(name.clone());- }- targets- .iter()- .find(|target| !excluded.contains(*target))- .cloned()- .ok_or(LibsyError::AllTargetsExcluded)-}--/// Key for overflow history: a root request by its session, a child request by its session-/// and agent. Keying a child finer than its session keeps one child's overflow from evicting-/// a target for the parent or a sibling sharing the session.+/// Key for routing affinity: a root request by its session, a child request by its session+/// and agent.#[derive(Clone, Hash, PartialEq, Eq)]pub(crate) enum RoutingIdentity {/// Root request, keyed by session ID.@@ -360,155 +330,6 @@ impl RoutingIdentity {Some(Self::Session(session.to_string()))}}-- /// The session this identity belongs to; shared by a session's root and its children.- fn session(&self) -> &str {- match self {- Self::Session(session) | Self::Subagent { session, .. } => session,- }- }-}--/// Bounds process-local overflow history. Dropping a live entry costs one rediscovered-/// overflow, so the victim choice does not need to be exact.-const MAX_EVICTION_IDENTITIES: usize = 1_024;--/// Per-identity record of the targets that overflowed their context window.-///-/// A conversation only grows, so a target that could not fit one turn will not fit a-/// later one; remembering it lets the next turn skip a call certain to fail. Requests-/// without a routing identity are not tracked — there is nothing to remember them by.-#[derive(Default)]-pub(crate) struct SessionEvictions {- by_identity: Mutex<HashMap<RoutingIdentity, HashSet<ModelId>>>,-}--impl SessionEvictions {- /// Forgets overflow history for a completed session, including every child of it.- pub(crate) fn remove_session(&self, session: &str) {- self.by_identity- .lock()- .retain(|identity, _| identity.session() != session);- }-- /// The targets `identity` has already overflowed; empty for an untracked request.- fn evicted_for(&self, identity: Option<&RoutingIdentity>) -> Vec<ModelId> {- let Some(identity) = identity else {- return Vec::new();- };- self.by_identity- .lock()- .get(identity)- .map(|targets| targets.iter().cloned().collect())- .unwrap_or_default()- }-- /// Remembers that `target` overflowed for `identity`, tracking at most- /// [`MAX_EVICTION_IDENTITIES`] identities.- fn record(&self, identity: Option<&RoutingIdentity>, target: &ModelId) {- let Some(identity) = identity else { return };- let mut histories = self.by_identity.lock();- if histories.len() >= MAX_EVICTION_IDENTITIES- && !histories.contains_key(identity)- && let Some(oldest) = histories.keys().next().cloned()- {- histories.remove(&oldest);- }- histories- .entry(identity.clone())- .or_default()- .insert(target.clone());- }-}--/// How many of `targets` this request is still allowed to reach.-fn eligible_targets(targets: &[ModelId], excluded: &HashSet<ModelId>) -> usize {- targets- .iter()- .filter(|target| !excluded.contains(*target))- .count()-}--/// Bars the targets `identity` has already overflowed from this request, so routing does-/// not select one that is certain to fail again.-pub(crate) fn exclude_evicted(- excluded: &mut HashSet<ModelId>,- targets: &[ModelId],- evictions: &SessionEvictions,- identity: Option<&RoutingIdentity>,-) {- for target in evictions.evicted_for(identity) {- // Never seed the pool empty: a later turn may be small enough to serve, and the- // caller should get the upstream's answer rather than a routing error.- if eligible_targets(targets, excluded) <= 1 {- break;- }- excluded.insert(target);- }-}--/// Returns the failed target and routing fallback policy for a terminal client error.-fn classify_fallback(error: &LibsyError) -> Option<(&ModelId, RoutingFallbackReason)> {- let LibsyError::ClientCall { target, source } = error else {- return None;- };- let reason = match source {- LlmClientError::ContextWindowExceeded { .. } => RoutingFallbackReason::ContextWindow,- LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => {- RoutingFallbackReason::Unavailable- }- LlmClientError::UpstreamHttp { status, .. }- if matches!(*status, 403 | 408 | 429) || (500..=599).contains(status) =>- {- RoutingFallbackReason::Unavailable- }- _ => return None,- };- Some((target, reason))-}--/// Calls `target`, falling back to the next eligible target after a route-level failure,-/// until a call succeeds or every target has been tried.-///-/// Routing is deliberately not re-run: the fallback replaces the target in place, so the-/// caller's request-side work and retained state still see exactly one turn.-/// `fallback_decision` builds the [`Decision`] published for a `from -> to` hop. Context-/// overflows are recorded for `identity`; unavailable targets remain request-local.-#[allow(clippy::too_many_arguments)]-pub(crate) async fn call_model_with_fallback(- excluded: &mut HashSet<ModelId>,- driver: &Driver,- targets: &[ModelId],- mut target: ModelId,- mut decision: Decision,- request: Request,- identity: Option<&RoutingIdentity>,- evictions: &SessionEvictions,- target_unavailable: impl Fn(&Request, &ModelId),- fallback_decision: impl Fn(&ModelId, &ModelId, RoutingFallbackReason) -> Decision,-) -> Result<Response> {- loop {- let result = driver.call_model(request.clone(), decision.clone()).await;- let Err(error) = result else { return result };- let Some((failed, reason)) = classify_fallback(&error) else {- return Err(error);- };- // A target already excluded means the pool is spent; surface the client error- // so the caller still sees the concrete upstream failure.- if !excluded.insert(failed.clone()) {- return Err(error);- }- match reason {- RoutingFallbackReason::ContextWindow => evictions.record(identity, failed),- RoutingFallbackReason::Unavailable => target_unavailable(&request, failed),- }- let Ok(next) = select_eligible_model(targets, &target, excluded) else {- return Err(error);- };- decision = fallback_decision(&target, &next, reason);- target = next;- driver.decide(decision.clone()).await?;- }}/// An optimization strategy. Implement [`route`](Self::route);@@ -588,6 +409,8 @@ pub trait Algorithm: Send + Sync + 'static {#[cfg(test)]mod tests {+ use std::collections::HashMap;+use super::*;use crate::core::testing::{Serve, ServeResult, echo, reply, test_drive};use futures::StreamExt;@@ -603,61 +426,6 @@ mod tests {LibsyError::external("test", TestError(message))}- fn classified_client_error(source: LlmClientError) -> Option<RoutingFallbackReason> {- classify_fallback(&LibsyError::client_call("target", source)).map(|(_, reason)| reason)- }-- #[test]- fn route_fallback_only_accepts_context_and_unavailable_failures() {- assert_eq!(- classified_client_error(LlmClientError::ContextWindowExceeded {- model: ModelId::from("target"),- message: "too long".to_string(),- }),- Some(RoutingFallbackReason::ContextWindow)- );- for source in [- LlmClientError::Transport {- source: Box::new(std::io::Error::other("connection failed")),- },- LlmClientError::Timeout {- source: Box::new(std::io::Error::other("request timed out")),- },- ] {- assert_eq!(- classified_client_error(source),- Some(RoutingFallbackReason::Unavailable)- );- }- for (status, expected) in [- (400, None),- (401, None),- (403, Some(RoutingFallbackReason::Unavailable)),- (404, None),- (408, Some(RoutingFallbackReason::Unavailable)),- (409, None),- (429, Some(RoutingFallbackReason::Unavailable)),- (499, None),- (500, Some(RoutingFallbackReason::Unavailable)),- (599, Some(RoutingFallbackReason::Unavailable)),- (600, None),- ] {- assert_eq!(- classified_client_error(LlmClientError::UpstreamHttp {- status,- body: "failed".to_string(),- }),- expected- );- }- assert_eq!(- classified_client_error(LlmClientError::InvalidResponse {- source: Box::new(std::io::Error::other("invalid response")),- }),- None- );- }-/// Build a routed decision for orchestration tests.fn test_decision(selected_model_id: ModelId) -> Decision {Decision::new(selected_model_id, true)@@ -683,7 +451,7 @@ mod tests {.clone();let decision = test_decision(target.clone());driver.decide(decision.clone()).await?;- driver.call_model(request, decision).await+ driver.call_model(request, vec![target], true).await}}@@ -713,12 +481,12 @@ mod tests {let first_driver = driver.clone();let mut first = tokio::spawn(async move {first_driver- .call_model(request(), test_decision(ModelId::from("first")))+ .call_model(request(), vec![ModelId::from("first")], true).await});let second = tokio::spawn(async move {driver- .call_model(request(), test_decision(ModelId::from("second")))+ .call_model(request(), vec![ModelId::from("second")], true).await});@@ -728,7 +496,11 @@ mod tests {let Step::CallModel(call) = step else {return Err(test_error("expected a CallModel step"));};- let selected_model = call.selected_model_id().to_string();+ let selected_model = call+ .models+ .first()+ .ok_or_else(|| test_error("model call has no candidates"))?+ .to_string();calls.insert(selected_model, call);}assert!(@@ -765,7 +537,7 @@ mod tests {let (driver, mut step_rx) = Driver::new("test");let producer = tokio::spawn(async move {driver- .call_model(request(), test_decision(ModelId::from("dropped")))+ .call_model(request(), vec![ModelId::from("dropped")], true).await});let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;@@ -803,19 +575,18 @@ mod tests {#[tokio::test]async fn into_parts_yields_the_selected_model_without_answering_it() -> Result<()> {let (driver, mut step_rx) = Driver::new("test");- let decision = test_decision(ModelId::from("answer/model"));- let producer = tokio::spawn({- let decision = decision.clone();- async move { driver.call_model(request(), decision).await }+ let producer = tokio::spawn(async move {+ driver+ .call_model(request(), vec![ModelId::from("answer/model")], true)+ .await});let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??;let Step::CallModel(call) = step else {return Err(test_error("expected a CallModel step"));};- let (taken_request, taken_decision) = call.into_parts();- assert_eq!(taken_decision.selected_model_id(), "answer/model");- assert!(taken_decision.is_answer_call());+ let (taken_request, taken_models) = call.into_parts();+ assert_eq!(taken_models, vec![ModelId::from("answer/model")]);assert_eq!(taken_request.llm_request.model.as_deref(),Some("answer/model")@@ -859,7 +630,7 @@ mod tests {/// replaying `chunks` in order (as `Ok` items).fn streaming_orch(chunks: Vec<LlmResponseChunk>) -> (Arc<dyn Algorithm>, impl Serve) {let algo = orch(target_set(&["stream/model"]));- let serve = move |_decision: Decision, _request: Request| {+ let serve = move |_target: ModelId, _request: Request| {let chunks = chunks.clone();async move {let stream =@@ -943,7 +714,7 @@ mod tests {match step? {Step::CallModel(call) => {saw_call = true;- assert_eq!(call.selected_model_id(), "offload/model");+ assert_eq!(call.models, vec![ModelId::from("offload/model")]);// Fulfilling the promise is the "real" model call the caller makes.call.respond(Ok(Response {llm_response: LlmResponse::Agg(text_response(@@ -1012,11 +783,11 @@ mod tests {for _ in 0..N {let algo = algo.clone();let barrier = barrier.clone();- let serve = move |decision: Decision, _request: Request| {+ let serve = move |target: ModelId, _request: Request| {let barrier = barrier.clone();async move {barrier.wait().await;- Ok(reply(decision.selected_model_id()))+ Ok(reply(target))}};handles.push(tokio::spawn(async move {@@ -1301,10 +1072,8 @@ mod tests {}async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {- let dec_w = test_decision(self.winner.clone().into());- let dec_l = test_decision(self.loser.clone().into());- let win = driver.call_model(request.clone(), dec_w);- let lose = driver.call_model(request, dec_l);+ let win = driver.call_model(request.clone(), vec![self.winner.clone().into()], true);+ let lose = driver.call_model(request, vec![self.loser.clone().into()], true);// First to resolve wins; `select!` drops the losing future (and its promise).tokio::select! {res = win => res,@@ -1322,10 +1091,10 @@ mod tests {winner: "winner".to_string(),loser: "loser".to_string(),});- let serve = move |decision: Decision, _request: Request| {+ let serve = move |target: ModelId, _request: Request| {let started = started.clone();async move {- if decision.selected_model_id() == "loser" {+ if target == "loser" {started.notify_one();match loser_delay {Some(delay) => tokio::time::sleep(delay).await,@@ -1334,7 +1103,7 @@ mod tests {} else {started.notified().await;}- Ok(reply(decision.selected_model_id()))+ Ok(reply(target))}};(algo, serve)@@ -1400,8 +1169,7 @@ mod tests {async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {let offloads = futures::future::join_all((0..self.n).map(|i| {- let decision = test_decision(format!("m{i}").into());- driver.call_model(request.clone(), decision)+ driver.call_model(request.clone(), vec![format!("m{i}").into()], true)}));tokio::select! {_ = offloads => Err(test_error("offloads unexpectedly completed")),@@ -1420,7 +1188,7 @@ mod tests {// Serving enters each call; once all N are in flight it signals, then pends forever.let started = Arc::new(AtomicUsize::new(0));- let serve = move |_decision: Decision, _request: Request| {+ let serve = move |_target: ModelId, _request: Request| {let started = started.clone();let all_started = all_started.clone();async move {diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rsindex 77191ae0e..a515a7c64 100644--- a/crates/libsy/src/core/classifier.rs+++ b/crates/libsy/src/core/classifier.rs@@ -76,11 +76,6 @@ pub trait Classifier<S = ()>: Send + Sync {None}- /// Drops retained routing state when `target` was unavailable for `request`.- ///- /// Stateless classifiers do not need to implement this hook.- fn target_unavailable(&self, _request: &Request, _target: &ModelId) {}-/// Score the classifier's targets given the current state and request.////// When present, `driver` lets a classifier offload model calls. It is `None`diff --git a/crates/libsy/src/core/testing.rs b/crates/libsy/src/core/testing.rsindex 1f03b74cc..deee05650 100644--- a/crates/libsy/src/core/testing.rs+++ b/crates/libsy/src/core/testing.rs@@ -17,7 +17,7 @@ use std::sync::Arc;use futures::future::BoxFuture;use switchyard_protocol::{- Decision, LlmClientError, LlmResponse, Request, Response, text_response,+ Decision, LlmClientError, LlmResponse, ModelId, Request, Response, text_response,};use crate::core::algorithm::{Algorithm, CallModel};@@ -29,16 +29,16 @@ pub(crate) type ServeResult = std::result::Result<Response, LlmClientError>;/// Answers offloaded model calls. Returning `Err` propagates a failed *model* call back into/// the algorithm, which may route around it.pub(crate) trait Serve: Send + Sync + 'static {- fn serve(&self, decision: Decision, request: Request) -> BoxFuture<'static, ServeResult>;+ fn serve(&self, target: ModelId, request: Request) -> BoxFuture<'static, ServeResult>;}impl<F, Fut> Serve for Fwhere- F: Fn(Decision, Request) -> Fut + Send + Sync + 'static,+ F: Fn(ModelId, Request) -> Fut + Send + Sync + 'static,Fut: Future<Output = ServeResult> + Send + 'static,{- fn serve(&self, decision: Decision, request: Request) -> BoxFuture<'static, ServeResult> {- Box::pin(self(decision, request))+ fn serve(&self, target: ModelId, request: Request) -> BoxFuture<'static, ServeResult> {+ Box::pin(self(target, request))}}@@ -59,10 +59,9 @@ pub(crate) async fn test_drive(/// error-shape assertions match production.async fn fulfill(serve: Arc<impl Serve>, call: CallModel) -> Result<()> {let request = call.request.clone();- let decision = call.decision.clone();- let target = decision.selected_model_id().to_string();+ let target = call.models.first().cloned().ok_or(LibsyError::NoTargets)?;let result = serve- .serve(decision, request)+ .serve(target.clone(), request).await.map_err(|source| LibsyError::client_call(target, source));call.respond(result)@@ -71,7 +70,7 @@ async fn fulfill(serve: Arc<impl Serve>, call: CallModel) -> Result<()> {/// Answers with the selected model name as the completion — what most routing tests need,/// since they assert on *which* target was called.pub(crate) fn echo() -> impl Serve {- |decision: Decision, _request: Request| async move { Ok(reply(decision.selected_model_id())) }+ |target: ModelId, _request: Request| async move { Ok(reply(target)) }}/// A buffered response whose completion text is `completion`.diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rsindex aa5c63b4a..7cb47b06d 100644--- a/crates/libsy/src/error.rs+++ b/crates/libsy/src/error.rs@@ -50,10 +50,6 @@ pub enum LibsyError {source: LlmClientError,},- /// Every target overflowed its context window.- #[error("every target exceeded its context window")]- AllTargetsExcluded,-/// A user extension or other foreign operation failed.#[error("{operation} failed: {source}")]External {diff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rsindex f4c4b257e..5c50e3341 100644--- a/crates/protocol/src/envelope.rs+++ b/crates/protocol/src/envelope.rs@@ -49,4 +49,40 @@ impl Response {pub fn selected_model(&self) -> Option<&str> {self.llm_response.selected_model()}++ /// Returns the Switchyard target that successfully served this response.+ ///+ /// Unlike [`Self::selected_model`], this is available for streamed responses because the+ /// client records the target when it receives the stream handle.+ pub fn served_model(&self) -> Option<&ModelId> {+ self.metadata.as_ref()?.served_model.as_ref()+ }++ /// Records the Switchyard target that successfully served this response.+ pub fn set_served_model(&mut self, model: &ModelId) {+ self.metadata.get_or_insert_default().served_model = Some(model.clone());+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::text_response;++ #[test]+ fn served_model_round_trips_through_response_metadata() {+ let mut response = Response {+ llm_response: LlmResponse::Agg(text_response(None, "answer")),+ metadata: None,+ };++ assert_eq!(response.served_model(), None);+ response.set_served_model(&ModelId::from("first"));+ assert_eq!(response.served_model().map(ModelId::as_str), Some("first"));+ response.set_served_model(&ModelId::from("fallback"));+ assert_eq!(+ response.served_model().map(ModelId::as_str),+ Some("fallback")+ );+ }}diff --git a/crates/protocol/src/metadata.rs b/crates/protocol/src/metadata.rsindex a9e11ed49..59f13122f 100644--- a/crates/protocol/src/metadata.rs+++ b/crates/protocol/src/metadata.rs@@ -9,7 +9,7 @@use std::{collections::BTreeMap, str::FromStr as _};-use crate::WireFormat;+use crate::{ModelId, WireFormat};// Dotted paths addressing fields inside Codex's turn-metadata header JSON value.const CODEX_SESSION_ID_PATH: &str = "x-codex-turn-metadata.session_id";@@ -185,6 +185,8 @@ pub struct Metadata {pub session_final: Option<bool>,/// External trace/request id for joining with the host's telemetry.pub correlation_id: Option<String>,+ /// Switchyard target that successfully served a response.+ pub served_model: Option<ModelId>,/// Arbitrary host-defined key/value metadata.pub extra_metadata: Option<BTreeMap<String, String>>,/// HTTP headers to attach when forwarding the request/response, if any.diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 78a095260..f52e29e36 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -184,15 +184,26 @@ struct PyModelCall {inner: Option<CallModel>,algorithm: String,request: Py<PyAny>,+ models: Vec<String>,decision: Py<PyDecision>,}impl PyModelCall {fn new(py: Python<'_>, call: CallModel) -> PyResult<Self> {let request = to_python(py, &call.request.llm_request)?;- let decision = Py::new(py, PyDecision::from(call.decision.clone()))?;+ let selected = call+ .models+ .first()+ .cloned()+ .ok_or(RustLibsyError::NoTargets)+ .map_err(py_libsy_error)?;+ let decision = Py::new(+ py,+ PyDecision::from(Decision::new(selected, call.is_answer_call)),+ )?;Ok(Self {algorithm: call.algorithm.clone(),+ models: call.models.iter().map(ToString::to_string).collect(),inner: Some(call),request,decision,@@ -220,6 +231,12 @@ impl PyModelCall {self.request.clone_ref(py)}+ /// Candidate models in the order the host should try them.+ #[getter]+ fn models(&self) -> Vec<String> {+ self.models.clone()+ }+/// The routing decision behind this call.#[getter]fn decision(&self, py: Python<'_>) -> Py<PyDecision> {@@ -229,11 +246,9 @@ impl PyModelCall {/// Consume the answer call without serving it and return its rewritten request and decision.#[pyo3(name = "into_parts")]fn take_parts(&mut self, py: Python<'_>) -> PyResult<(Py<PyAny>, Py<PyDecision>)> {- let (request, decision) = self.take()?.into_parts();- Ok((- to_python(py, &request.llm_request)?,- Py::new(py, PyDecision::from(decision))?,- ))+ let decision = self.decision.clone_ref(py);+ let (request, _models) = self.take()?.into_parts();+ Ok((to_python(py, &request.llm_request)?, decision))}/// Fulfill this call with an aggregate normalized response dictionary.@@ -254,7 +269,12 @@ impl PyModelCall {return Err(PyTypeError::new_err("error must derive from BaseException"));}let call = self.take()?;- let target = call.decision.selected_model_id().clone();+ let target = call+ .models+ .first()+ .cloned()+ .ok_or(RustLibsyError::NoTargets)+ .map_err(py_libsy_error)?;let source = if error.is_instance_of::<ContextWindowExceededError>() {LlmClientError::ContextWindowExceeded {model: target.clone(),diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex b1eb5acfb..fd02cd01b 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -37,7 +37,7 @@ use parking_lot::Mutex;use serde::Deserialize;use serde_json::{Value, json};use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient};-use switchyard_protocol::{Decision, LlmClientError, Metadata, ModelId, Request, Usage};+use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage};use tokio::net::{TcpListener, TcpSocket};use tokio::task;use tracing::{Instrument, Level};@@ -757,22 +757,21 @@ async fn handle_llm_request(Ok(result) => result,Err(error) => return algorithm_error(error),};- // Metrics, response body, and routing header all read the same decision, so- // the model they name can never disagree. An empty trace leaves the body with- // the id the upstream reported.let decision = trace.last();- let response = if let Some(decision) = decision {+ // The response carries the candidate that actually served it. Fall back to the routing+ // decision for algorithms that return a response without an offloaded model call.+ let served_model = response+ .served_model()+ .cloned()+ .or_else(|| decision.map(|decision| decision.selected_model_id().clone()));+ let response = if let Some(served_model) = served_model.as_ref() {let cache_eligible = cache_probe.as_ref()- .map(|probe| {- state- .stats- .prefix_eligibility(decision.selected_model_id(), probe)- })+ .map(|probe| state.stats.prefix_eligibility(served_model, probe)).unwrap_or(0.0);usage_metrics::observe(response,- decision.selected_model_id(),+ served_model.as_str(),started.0,state.stats,cache_eligible,@@ -782,13 +781,13 @@ async fn handle_llm_request(response};- let served_model = decision.map(|decision| decision.selected_model_id().to_string());- let mut response = match into_http_response(response, wire_format, served_model) {+ let response_model = served_model.as_ref().map(ToString::to_string);+ let mut response = match into_http_response(response, wire_format, response_model) {Ok(response) => response,Err(error) => return server_error(error.to_string()),};- if let Some(decision) = decision {- attach_routing_headers(&mut response, decision);+ if let Some(served_model) = served_model.as_ref() {+ attach_routing_headers(&mut response, served_model.as_str());}response}@@ -864,12 +863,8 @@ fn metadata_from_headers(headers: HeaderMap) -> Metadata {metadata}-fn attach_routing_headers(response: &mut Response, decision: &Decision) {- insert_routing_header(- response,- HEADER_SELECTED_MODEL,- decision.selected_model_id(),- );+fn attach_routing_headers(response: &mut Response, served_model: &str) {+ insert_routing_header(response, HEADER_SELECTED_MODEL, served_model);}fn insert_routing_header(response: &mut Response, name: &'static str, value: &str) {diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex b3cfb3791..1cd87dca7 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -1998,96 +1998,6 @@ async fn routing_log_keeps_the_canonical_session_id_until_a_stream_drains() -> TOk(())}-// Overflow history is isolated per child, cleared with the session, and not retained when a-// child lacks an agent ID.-#[tokio::test]-async fn overflow_history_is_scoped_to_agent_and_session_lifetime() -> TestResult {- let upstream = MockUpstream::start().await?;- let state = fallback_state(&upstream.base_url)?;- let app = build_switchyard_router(state);- let child_a = [- ("x-switchyard-session-id", "shared-session"),- ("x-switchyard-agent-id", "child-a"),- ("x-switchyard-is-subagent", "true"),- ];- let root = [- ("x-switchyard-session-id", "shared-session"),- ("x-switchyard-agent-id", "root"),- ("x-switchyard-is-subagent", "false"),- ];- let child_b = [- ("x-switchyard-session-id", "shared-session"),- ("x-switchyard-agent-id", "child-b"),- ("x-switchyard-is-subagent", "true"),- ];- let child_without_agent_id = [- ("x-switchyard-session-id", "shared-session"),- ("x-switchyard-is-subagent", "true"),- ];- let final_root = [- ("x-switchyard-session-id", "shared-session"),- ("x-switchyard-agent-id", "root"),- ("x-switchyard-is-subagent", "false"),- ("x-switchyard-session-final", "true"),- ];- type Case<'a> = (&'a str, &'a [(&'a str, &'a str)], &'a [&'a str]);- let cases: [Case<'_>; 8] = [- (- "overflow",- child_a.as_slice(),- &["model/weak", "model/strong"],- ),- ("fits", child_a.as_slice(), &["model/strong"]),- ("fits", root.as_slice(), &["model/weak"]),- ("fits", child_b.as_slice(), &["model/weak"]),- ("fits", final_root.as_slice(), &["model/weak"]),- ("fits", child_a.as_slice(), &["model/weak"]),- (- "overflow",- child_without_agent_id.as_slice(),- &["model/weak", "model/strong"],- ),- (- "overflow",- child_without_agent_id.as_slice(),- &["model/weak", "model/strong"],- ),- ];-- for (content, headers, expected_calls) in cases {- let previous_call_count = upstream.calls.lock().await.len();- let response = send_with_headers(- &app,- "POST",- "/v1/chat/completions",- Some(json!({- "model": ROUTE_MODEL,- "messages": [{"role": "user", "content": content}]- })),- headers,- )- .await?;- assert_eq!(response.status, StatusCode::OK);- let expected_model = expected_calls.last().copied();- assert_eq!(- response- .headers- .get("x-model-router-selected-model")- .and_then(|value| value.to_str().ok()),- expected_model- );- let calls = upstream.calls.lock().await;- assert_eq!(- calls[previous_call_count..]- .iter()- .map(|call| call["model"].as_str().unwrap_or(""))- .collect::<Vec<_>>(),- expected_calls- );- }- Ok(())-}-#[tokio::test]async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted() -> TestResult {let upstream = MockUpstream::start().await?;@@ -2128,6 +2038,7 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted.and_then(|value| value.to_str().ok()),Some("model/strong"));+ assert_eq!(response.json()?["model"], "model/strong");let calls = upstream.calls.lock().await;assert_eq!(calls[previous_call_count..]@@ -2142,6 +2053,8 @@ async fn unavailable_target_fails_over_across_endpoints_and_stops_when_exhausted// Fallback causes are logged rather than accumulated in the legacy stats counters.assert_eq!(stats["routing_fallbacks"]["unavailable"], 0);assert_eq!(stats["routing_fallbacks"]["context_window"], 0);+ assert_eq!(stats["models"]["model/strong"]["calls"], 3);+ assert_eq!(stats["models"]["model/weak"]["errors"], 3);let records = std::fs::read_to_string(&log_path)?;let records = recordsdiff --git a/docs/operations/context_window.md b/docs/operations/context_window.mdindex 4ade38066..bd0e6b0b8 100644--- a/docs/operations/context_window.md+++ b/docs/operations/context_window.md@@ -1,14 +1,11 @@# Context-Window HandlingWhen an upstream rejects a request because the prompt exceeds the model's-context window, Switchyard drops that target and calls another target on the-same route, repeating until a call succeeds or every target has been tried. If-the request carries `x-switchyard-session-id` or a recognized coding-agent-session header such as `x-claude-code-session-id`, the target remains excluded-for the rest of that session. Without a session header, the fallback still-applies to the current request, but the overflow is not remembered.-After truncating or resetting context, clients should use a new session ID;-reusing the old ID preserves its target exclusions.+context window, Switchyard calls the remaining targets on the same route in+configured order, stopping when one answers or every target has been tried.+Fallback applies only to the current request. An overflow is not remembered+across turns, so the route may select and try the same target again on the next+request.## What counts as an overflow@@ -50,12 +47,9 @@ picker = "efficient_first"confidence_threshold = 0.5```-Response headers report where the request actually landed:--```text-x-model-router-selected-model: openai/gpt-4o-mini-x-model-router-rationale: openai/gpt-4o exceeded its context window; fell back to openai/gpt-4o-mini-```+The response body, `x-model-router-selected-model` header, usage metrics, and+routing log name the candidate that actually served the request, including when+the client falls through from the algorithm's first choice.## When every target overflowsdiff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.mdindex 7bf0da7bb..f4e4c451e 100644--- a/docs/routing_algorithms/stage_router_routing.md+++ b/docs/routing_algorithms/stage_router_routing.md@@ -9,8 +9,8 @@ mechanical work. Which tier a turn defaults to depends on the picker you chooseoff that default. You configure it with a single knob, `confidence_threshold`,plus an optional LLM classifier.-If the selected target exceeds its context window, the router tries the next-eligible target until one succeeds or all configured targets have been tried. See+If the selected target exceeds its context window, the client tries the route's+remaining targets in configured order for that request. See[Context-Window Handling](../operations/context_window.md).## How it worksdiff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex 498eeca05..67e4e1888 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -58,6 +58,9 @@ if TYPE_CHECKING:@propertydef request(self) -> dict[str, object]: ...+ @property+ def models(self) -> list[str]: ...+@propertydef decision(self) -> Decision: ...diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex 15b7b5843..8e93cd6bb 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -60,14 +60,20 @@ async def run_algorithm(async for step in algorithm.run_stream(request or request_body(), headers=headers):match step:case Step.CallModel(call):- target = call.decision.selected_model_id- client = (clients or {})[target]- try:- response = await client.call(call.request)- except Exception as error:- call.fail(error)- else:- call.respond(response)+ for index, target in enumerate(call.models):+ candidate_request = {**call.request, "model": target}+ client = (clients or {})[target]+ try:+ response = await client.call(candidate_request)+ except ContextWindowExceededError as error:+ if index + 1 == len(call.models):+ call.fail(error)+ except Exception as error:+ call.fail(error)+ break+ else:+ call.respond(response)+ breakcase Step.Decision(decision):decisions.append(decision)case Step.Done(response):@@ -86,6 +92,7 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()match step:case Step.CallModel(call):variants.append("call_model")+ assert call.models == ["fast"]client_response = await client.call(call.request)call.respond(client_response)with pytest.raises(LibsyError, match="already been completed"):@@ -118,6 +125,7 @@ async def test_into_parts_supports_decision_only_routing() -> None:case Step.CallModel(call) if call.decision.is_answer_call:request, decision = call.into_parts()assert call.algorithm == "random"+ assert call.models == ["fast"]assert request["messages"] == request_body()["messages"]assert decision.selected_model_id == "fast"assert decision.is_answer_call is True@@ -351,5 +359,5 @@ async def test_context_window_failure_falls_back_to_the_next_model() -> None:{"fast": OverflowClient(), "strong": EchoClient("strong")},)- assert [decision.selected_model_id for decision in decisions] == ["fast", "strong"]+ assert [decision.selected_model_id for decision in decisions] == ["fast"]assert response["model"] == "strong"