Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

chore(libsy): move call_with_overflow_fallback to util function (#222)

Greg Clark577331771824
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index a2aba53f8..489587300 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -16,27 +16,21 @@
//! Every composition retains one thing regardless: a target that overflows its context window is
//! remembered for the rest of its session and skipped on later turns.
-use std::{
- collections::{HashMap, HashSet},
- sync::Arc,
-};
+use std::{collections::HashMap, sync::Arc};
use async_trait::async_trait;
use parking_lot::Mutex;
use tokio::sync::Mutex as AsyncMutex;
+use crate::core::algorithm::{call_llm_with_overflow_fallback, exclude_evicted, SessionEvictions};
use crate::core::{Classification, Classifier, Event, Processor, Score};
use crate::{
- Algorithm, Context, Decision, Driver, LibsyError, LlmClientError, LlmTarget, LlmTargetSet,
- Request, Response, Result, RoutedLlmClient,
+ Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,
+ Result, RoutedLlmClient,
};
type SessionStates<S> = Mutex<HashMap<String, Arc<AsyncMutex<S>>>>;
-/// Matches the cap the Python stack uses. Dropping a live session's entry costs one
-/// rediscovered overflow, so the victim choice does not need to be exact.
-const MAX_EVICTION_SESSIONS: usize = 1_024;
-
/// The decision a fall-through run produces: the selected model plus a human-readable reason.
pub struct FallThroughDecision {
/// Target selected by the classifier cascade.
@@ -112,7 +106,7 @@ pub struct FallThrough<S = ()> {
classifiers: Vec<Arc<dyn Classifier<S>>>,
targets: LlmTargetSet,
session_states: Option<SessionStates<S>>,
- session_evictions: Mutex<HashMap<String, HashSet<String>>>,
+ session_evictions: SessionEvictions,
}
impl FallThrough<()> {
@@ -125,7 +119,7 @@ impl FallThrough<()> {
classifiers: Vec::new(),
targets,
session_states: None,
- session_evictions: Mutex::new(HashMap::new()),
+ session_evictions: SessionEvictions::default(),
}
}
}
@@ -143,7 +137,7 @@ where
classifiers: Vec::new(),
targets,
session_states: Some(Mutex::new(HashMap::new())),
- session_evictions: Mutex::new(HashMap::new()),
+ session_evictions: SessionEvictions::default(),
}
}
@@ -182,14 +176,12 @@ where
let mut request = request;
let session = session_id(&request);
let mut ctx = ctx;
- for target in self.evicted_in_session(session.as_deref()) {
- // Never seed the pool empty: a later turn may be small enough to serve, and the
- // caller should get the upstream's answer rather than a routing error.
- if self.eligible_targets(&ctx) <= 1 {
- break;
- }
- ctx.exclude_target(target);
- }
+ exclude_evicted(
+ &mut ctx,
+ &self.targets,
+ &self.session_evictions,
+ session.as_deref(),
+ );
let session_state = self.session_state(&request);
let (target, decision, served) = match session_state {
Some(state) => {
@@ -210,93 +202,22 @@ where
match served {
Some(response) => Ok(response),
None => {
- self.call_with_overflow_fallback(
+ call_llm_with_overflow_fallback(
ctx,
&driver,
+ &self.targets,
target,
decision,
request,
session.as_deref(),
+ &self.session_evictions,
+ |from, to| self.fallback_decision(from, to),
)
.await
}
}
}
- fn eligible_targets(&self, ctx: &Context) -> usize {
- self.targets
- .targets()
- .iter()
- .filter(|t| !ctx.is_excluded(&t.semantic_name))
- .count()
- }
-
- fn evicted_in_session(&self, session: Option<&str>) -> Vec<String> {
- let Some(session) = session else {
- return Vec::new();
- };
- self.session_evictions
- .lock()
- .get(session)
- .map(|targets| targets.iter().cloned().collect())
- .unwrap_or_default()
- }
-
- fn record_eviction(&self, session: Option<&str>, target: &str) {
- let Some(session) = session else { return };
- let mut sessions = self.session_evictions.lock();
- if sessions.len() >= MAX_EVICTION_SESSIONS && !sessions.contains_key(session) {
- if let Some(oldest) = sessions.keys().next().cloned() {
- sessions.remove(&oldest);
- }
- }
- sessions
- .entry(session.to_string())
- .or_default()
- .insert(target.to_string());
- }
-
- /// Calls `target`, falling back to the next eligible target whenever one overflows its
- /// context window, until a call succeeds or every target has been tried.
- ///
- /// Routing is deliberately not re-run: the fallback replaces the target in place so the
- /// processor chain and the retained session state still see exactly one turn.
- async fn call_with_overflow_fallback(
- &self,
- mut ctx: Context,
- driver: &Driver,
- mut target: LlmTarget,
- mut decision: Arc<dyn Decision>,
- request: Request,
- session: Option<&str>,
- ) -> Result<Response> {
- loop {
- let result = driver
- .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())
- .await;
- let Err(error) = result else { return result };
- let LibsyError::ClientCall {
- target: failed,
- source: LlmClientError::ContextWindowExceeded { .. },
- } = &error
- else {
- return Err(error);
- };
- // A target already excluded means the pool is spent; surface the client error
- // so the caller still sees a context overflow rather than an internal failure.
- if !ctx.exclude_target(failed) {
- return Err(error);
- }
- self.record_eviction(session, failed);
- let Ok(next) = self.targets.resolve_target(&target.semantic_name, &ctx) else {
- return Err(error);
- };
- decision = self.fallback_decision(&target, &next);
- target = next;
- driver.info(ctx.clone(), decision.clone()).await?;
- }
- }
-
/// The decision published when an overflow sends the request to a different target.
fn fallback_decision(&self, from: &LlmTarget, to: &LlmTarget) -> Arc<dyn Decision> {
Arc::new(FallThroughDecision {
@@ -438,7 +359,7 @@ mod tests {
completion_text, text_request, text_response, LlmRequest, Message, Metadata, Role,
};
- use crate::{LlmResponse, LlmTarget, RoutedLlmClient};
+ use crate::{LlmClientError, LlmResponse, LlmTarget, RoutedLlmClient};
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index c99c1802f..84eb41d55 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -6,6 +6,7 @@
//! calls and publishes [`Decision`]s over. See the crate root for the narrative model.
use std::{
+ collections::{HashMap, HashSet},
pin::Pin,
sync::Arc,
time::{Duration, Instant},
@@ -410,6 +411,123 @@ impl LlmTargetSet {
}
}
+/// Matches the cap the Python stack uses. Dropping a live session's entry costs one
+/// rediscovered overflow, so the victim choice does not need to be exact.
+const MAX_EVICTION_SESSIONS: usize = 1_024;
+
+/// Per-session record of the targets that overflowed their context window.
+///
+/// A conversation only grows, so a target that could not fit one turn will not fit a
+/// later one; remembering it lets the next turn skip a call certain to fail. Requests
+/// without a session id are not tracked — there is nothing to remember them by.
+#[derive(Default)]
+pub(crate) struct SessionEvictions {
+ sessions: Mutex<HashMap<String, HashSet<String>>>,
+}
+
+impl SessionEvictions {
+ /// The targets `session` has already overflowed; empty for an untracked request.
+ fn evicted_in(&self, session: Option<&str>) -> Vec<String> {
+ let Some(session) = session else {
+ return Vec::new();
+ };
+ self.sessions
+ .lock()
+ .get(session)
+ .map(|targets| targets.iter().cloned().collect())
+ .unwrap_or_default()
+ }
+
+ /// Remembers that `target` overflowed in `session`, tracking at most
+ /// [`MAX_EVICTION_SESSIONS`] sessions.
+ fn record(&self, session: Option<&str>, target: &str) {
+ let Some(session) = session else { return };
+ let mut sessions = self.sessions.lock();
+ if sessions.len() >= MAX_EVICTION_SESSIONS && !sessions.contains_key(session) {
+ if let Some(oldest) = sessions.keys().next().cloned() {
+ sessions.remove(&oldest);
+ }
+ }
+ sessions
+ .entry(session.to_string())
+ .or_default()
+ .insert(target.to_string());
+ }
+}
+
+/// How many of `targets` this request is still allowed to reach.
+fn eligible_targets(targets: &LlmTargetSet, ctx: &Context) -> usize {
+ targets
+ .targets()
+ .iter()
+ .filter(|t| !ctx.is_excluded(&t.semantic_name))
+ .count()
+}
+
+/// Bars the targets `session` has already overflowed from this request, so routing does
+/// not select one that is certain to fail again.
+pub(crate) fn exclude_evicted(
+ ctx: &mut Context,
+ targets: &LlmTargetSet,
+ evictions: &SessionEvictions,
+ session: Option<&str>,
+) {
+ for target in evictions.evicted_in(session) {
+ // Never seed the pool empty: a later turn may be small enough to serve, and the
+ // caller should get the upstream's answer rather than a routing error.
+ if eligible_targets(targets, ctx) <= 1 {
+ break;
+ }
+ ctx.exclude_target(target);
+ }
+}
+
+/// Calls `target`, falling back to the next eligible target in `targets` whenever one
+/// overflows its context window, until a call succeeds or every target has been tried.
+///
+/// Routing is deliberately not re-run: the fallback replaces the target in place, so the
+/// caller's request-side work and retained state still see exactly one turn.
+/// `fallback_decision` builds the [`Decision`] published for a `from -> to` hop, and each
+/// overflow is recorded against `session` so later turns skip that target outright.
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn call_llm_with_overflow_fallback(
+ mut ctx: Context,
+ driver: &Driver,
+ targets: &LlmTargetSet,
+ mut target: LlmTarget,
+ mut decision: Arc<dyn Decision>,
+ request: Request,
+ session: Option<&str>,
+ evictions: &SessionEvictions,
+ fallback_decision: impl Fn(&LlmTarget, &LlmTarget) -> Arc<dyn Decision>,
+) -> Result<Response> {
+ loop {
+ let result = driver
+ .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())
+ .await;
+ let Err(error) = result else { return result };
+ let LibsyError::ClientCall {
+ target: failed,
+ source: LlmClientError::ContextWindowExceeded { .. },
+ } = &error
+ else {
+ return Err(error);
+ };
+ // A target already excluded means the pool is spent; surface the client error
+ // so the caller still sees a context overflow rather than an internal failure.
+ if !ctx.exclude_target(failed) {
+ return Err(error);
+ }
+ evictions.record(session, failed);
+ let Ok(next) = targets.resolve_target(&target.semantic_name, &ctx) else {
+ return Err(error);
+ };
+ decision = fallback_decision(&target, &next);
+ target = next;
+ driver.info(ctx.clone(), decision.clone()).await?;
+ }
+}
+
/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);
/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)
/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).