Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): evict and reroute on context-window overflow (#187)
ryan-lempka27a9af010711
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 8a6e70fdb..958e6d19f 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -21,8 +21,8 @@ use tokio::sync::Mutex as AsyncMutex;use crate::core::{Classifier, Event, Processor, Score};use crate::{- Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,- RoutedLlmClient,+ Algorithm, Context, Decision, Driver, LibsyError, LlmClientError, LlmTarget, LlmTargetSet,+ Request, Response, Result, RoutedLlmClient,};type SessionStates<S> = Mutex<HashMap<String, Arc<AsyncMutex<S>>>>;@@ -157,11 +157,64 @@ where}};- driver- .call_llm_target(ctx, &target, request, decision)+ self.call_with_overflow_fallback(ctx, &driver, target, decision, request).await}+ /// Calls `target`, falling back to the next eligible target whenever one overflows its+ /// context window, until a call succeeds or every target has been tried.+ ///+ /// Routing is deliberately not re-run: the fallback replaces the target in place so the+ /// processor chain and the retained session state still see exactly one turn.+ async fn call_with_overflow_fallback(+ &self,+ mut ctx: Context,+ driver: &Driver,+ mut target: LlmTarget,+ mut decision: Arc<dyn Decision>,+ request: Request,+ ) -> Result<Response> {+ loop {+ let result = driver+ .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())+ .await;+ let Err(error) = result else { return result };+ let LibsyError::ClientCall {+ target: failed,+ source: LlmClientError::ContextWindowExceeded { .. },+ } = &error+ else {+ return Err(error);+ };+ // A target already excluded means the pool is spent; surface the client error+ // so the caller still sees a context overflow rather than an internal failure.+ if !ctx.exclude_target(failed) {+ return Err(error);+ }+ let Ok(next) = self.targets.resolve_target(&target.semantic_name, &ctx) else {+ return Err(error);+ };+ decision = self.fallback_decision(&target, &next);+ target = next;+ driver.info(ctx.clone(), decision.clone()).await?;+ }+ }++ /// The decision published when an overflow sends the request to a different target.+ fn fallback_decision(&self, from: &LlmTarget, to: &LlmTarget) -> Arc<dyn Decision> {+ Arc::new(FallThroughDecision {+ selected_model: to.semantic_name.clone(),+ reasoning: format!(+ "{} exceeded its context window; fell back to {}",+ from.semantic_name, to.semantic_name+ ),+ tier: self+ .classifiers+ .iter()+ .find_map(|c| c.routing_tier(&to.semantic_name)),+ })+ }+/// Returns this request's retained state without holding the registry lock.fn session_state(&self, request: &Request) -> Option<Arc<AsyncMutex<S>>> {let states = self.session_states.as_ref()?;@@ -193,12 +246,12 @@ where// 2. Fall through the cascade: the first classifier to score decides (argmax). The// per-request driver is offered to each — driver-backed classifiers use it.let mut maybe_score: Option<Score> = None;- let mut maybe_tier: Option<&'static str> = None;+ let mut deciding: Option<&Arc<dyn Classifier<S>>> = None;for classifier in &self.classifiers {let scores = classifier.score(state, request, Some(driver)).await?;maybe_score = scores.argmax(false)?;- if let Some(s) = maybe_score.as_ref() {- maybe_tier = classifier.routing_tier(&s.target);+ if maybe_score.is_some() {+ deciding = Some(classifier);break;}}@@ -208,12 +261,21 @@ where});};- // 3. Resolve the target and publish the decision.- let target = self.targets.get_target(&score.target)?;+ // 3. Resolve the target and publish the decision. When an excluded target sends+ // the request elsewhere, the tier and reasoning describe where it actually went.+ let target = self.targets.resolve_target(&score.target, ctx)?;+ let reasoning = if target.semantic_name == score.target {+ (self.decision_reason)(&self.name, &score)+ } else {+ format!(+ "{} exceeded its context window; fell back to {}",+ score.target, target.semantic_name+ )+ };let decision: Arc<dyn Decision> = Arc::new(FallThroughDecision {- selected_model: score.target.clone(),- reasoning: (self.decision_reason)(&self.name, &score),- tier: maybe_tier,+ selected_model: target.semantic_name.clone(),+ reasoning,+ tier: deciding.and_then(|c| c.routing_tier(&target.semantic_name)),});driver.info(ctx.clone(), decision.clone()).await?;@@ -423,6 +485,144 @@ mod tests {// --- tests -------------------------------------------------------------------------+ /// Overflows for the named targets and echoes for the rest.+ struct OverflowClient {+ overflowing: Vec<&'static str>,+ }++ #[async_trait]+ impl RoutedLlmClient for OverflowClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, LlmClientError> {+ let model = decision.selected_model().to_string();+ if self.overflowing.contains(&model.as_str()) {+ return Err(LlmClientError::ContextWindowExceeded {+ model,+ message: "prompt is too long".to_string(),+ });+ }+ Ok(Response {+ llm_response: LlmResponse::Agg(text_response(None, model)),+ metadata: None,+ })+ }+ }++ /// `target_set`, but the named `overflowing` targets reject every call with a+ /// context-window error so the retry path can be driven.+ fn target_set_with_overflow(names: &[&str], overflowing: &[&'static str]) -> LlmTargetSet {+ LlmTargetSet::new(+ names+ .iter()+ .map(|name| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(Arc::new(OverflowClient {+ overflowing: overflowing.to_vec(),+ }) as Arc<dyn RoutedLlmClient>),+ })+ .collect(),+ )+ }++ #[tokio::test]+ async fn an_overflowing_target_is_retried_on_one_that_fits() -> Result<()> {+ let router =+ FallThrough::<()>::new(target_set_with_overflow(&["weak", "strong"], &["weak"]))+ .with_classifier(fixed(vec![score("weak", 0.9)]));+ let (model, _) = run(router).await?;+ assert_eq!(model, "strong");+ Ok(())+ }++ #[tokio::test]+ async fn overflowing_targets_are_retried_until_one_fits() -> Result<()> {+ let router = FallThrough::<()>::new(target_set_with_overflow(+ &["weak", "mid", "strong"],+ &["weak", "mid"],+ ))+ .with_classifier(fixed(vec![score("weak", 0.9)]));+ let (model, _) = run(router).await?;+ assert_eq!(model, "strong");+ Ok(())+ }++ #[tokio::test]+ async fn exhausting_every_target_surfaces_the_client_overflow() -> Result<()> {+ // Only the client error maps to a 400 upstream, so it must survive exhaustion.+ let router = FallThrough::<()>::new(target_set_with_overflow(+ &["weak", "strong"],+ &["weak", "strong"],+ ))+ .with_classifier(fixed(vec![score("weak", 0.9)]));+ match run(router).await {+ Ok(_) => panic!("expected an overflow error, got a response"),+ Err(LibsyError::ClientCall {+ source: LlmClientError::ContextWindowExceeded { .. },+ ..+ }) => Ok(()),+ Err(other) => panic!("expected ContextWindowExceeded, got {other:?}"),+ }+ }++ #[tokio::test]+ async fn a_retried_request_runs_the_processors_once() -> Result<()> {+ // Routing runs before the call loop, so an overflow must not replay processors.+ struct CountingProcessor(Arc<Mutex<Vec<&'static str>>>);++ #[async_trait]+ impl Processor for CountingProcessor {+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {+ let kind = match event {+ Event::Request(_) => "request",+ Event::Decision { .. } => "decision",+ _ => "other",+ };+ self.0.lock().push(kind);+ Ok(())+ }+ }++ let seen = Arc::new(Mutex::new(Vec::new()));+ let router =+ FallThrough::<()>::new(target_set_with_overflow(&["weak", "strong"], &["weak"]))+ .with_classifier(fixed(vec![score("weak", 0.9)]))+ .with_processor(Arc::new(CountingProcessor(seen.clone())));+ let (model, _) = run(router).await?;+ assert_eq!(model, "strong");+ assert_eq!(seen.lock().iter().filter(|e| **e == "request").count(), 1);+ assert_eq!(seen.lock().iter().filter(|e| **e == "decision").count(), 1);+ Ok(())+ }++ #[tokio::test]+ async fn an_excluded_target_reports_the_target_it_fell_back_to() -> Result<()> {+ // Headers and usage metrics read the decision, so it must describe the real call.+ let router = Arc::new(+ FallThrough::<()>::new(target_set(&["weak", "strong"]))+ .with_classifier(fixed(vec![score("weak", 0.9)])),+ );+ let mut ctx = Context::default();+ ctx.exclude_target("weak");+ let (trace, response) = router.run(ctx, request()).await?;+ let text = response+ .llm_response+ .into_agg()+ .await+ .map(|agg| completion_text(&agg))+ .map_err(|error| LibsyError::external("aggregating fall-through response", error))?;++ assert_eq!(text, "strong");+ assert_eq!(trace[0].selected_model(), "strong");+ assert!(trace[0]+ .reasoning()+ .is_some_and(|r| r.contains("fell back to strong")));+ Ok(())+ }+#[tokio::test]async fn argmax_picks_the_highest_confidence_target() -> Result<()> {let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 5704aab5f..81135dab2 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -327,6 +327,20 @@ impl LlmTargetSet {})}+ /// The named target, or the first one this request is not barred from when it has been+ /// excluded (see [`Context::exclude_target`]). Errors if every target is excluded.+ pub fn resolve_target(&self, name: &str, ctx: &Context) -> Result<LlmTarget> {+ let target = self.get_target(name)?;+ if !ctx.is_excluded(&target.semantic_name) {+ return Ok(target);+ }+ self.targets+ .iter()+ .find(|t| !ctx.is_excluded(&t.semantic_name))+ .cloned()+ .ok_or(LibsyError::AllTargetsExcluded)+ }+/// The first target's client that can serve `count_tokens` (an Anthropic/// upstream), or `None` when no target has one. Used by an algorithm's/// [`count_tokens_client`](crate::Algorithm::count_tokens_client).@@ -464,6 +478,7 @@ pub trait Algorithm: Send + Sync + 'static {/// Process a request to completion, returning the final [`Response`] and the trace of/// [`Decision`]s the algorithm made along the way.+ ///async fn run(self: Arc<Self>,ctx: Context,diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rsindex cb8fc70fa..834ceff33 100644--- a/crates/libsy/src/error.rs+++ b/crates/libsy/src/error.rs@@ -65,6 +65,10 @@ pub enum LibsyError {source: LlmClientError,},+ /// Every target overflowed its context window.+ #[error("every target exceeded its context window")]+ AllTargetsExcluded,+/// A user extension or other foreign operation failed.#[error("{operation} failed: {source}")]External {diff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rsindex 570f3873f..8399f936e 100644--- a/crates/protocol/src/envelope.rs+++ b/crates/protocol/src/envelope.rs@@ -5,13 +5,31 @@//! with the original provider payload and correlation [`Metadata`].use crate::{LlmRequest, LlmResponse, Metadata};-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};/// Cross-cutting request context passed through algorithms and LLM clients.#[derive(Clone, Debug, Default, PartialEq, Eq)]pub struct Context {/// Caller-specific values propagated through the request.pub values: HashMap<String, String>,+ /// Targets this request must not route to. A caller may set these up front to keep a+ /// request off a given target; routing also adds any target that overflows its context+ /// window mid-request.+ excluded_targets: HashSet<String>,+}++impl Context {+ /// Excludes a target from this request, returning whether it was newly excluded.+ /// Re-excluding returns `false`, which is what bounds the overflow retry once every+ /// target has been tried.+ pub fn exclude_target(&mut self, target: impl Into<String>) -> bool {+ self.excluded_targets.insert(target.into())+ }++ /// Whether this request is barred from routing to `target`.+ pub fn is_excluded(&self, target: &str) -> bool {+ self.excluded_targets.contains(target)+ }}/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget