Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
chore(libsy): move call_with_overflow_fallback to util function (#222)
Greg Clark577331771824
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex a2aba53f8..489587300 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -16,27 +16,21 @@//! Every composition retains one thing regardless: a target that overflows its context window is//! remembered for the rest of its session and skipped on later turns.-use std::{- collections::{HashMap, HashSet},- sync::Arc,-};+use std::{collections::HashMap, sync::Arc};use async_trait::async_trait;use parking_lot::Mutex;use tokio::sync::Mutex as AsyncMutex;+use crate::core::algorithm::{call_llm_with_overflow_fallback, exclude_evicted, SessionEvictions};use crate::core::{Classification, Classifier, Event, Processor, Score};use crate::{- Algorithm, Context, Decision, Driver, LibsyError, LlmClientError, LlmTarget, LlmTargetSet,- Request, Response, Result, RoutedLlmClient,+ Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,+ Result, RoutedLlmClient,};type SessionStates<S> = Mutex<HashMap<String, Arc<AsyncMutex<S>>>>;-/// Matches the cap the Python stack uses. Dropping a live session's entry costs one-/// rediscovered overflow, so the victim choice does not need to be exact.-const MAX_EVICTION_SESSIONS: usize = 1_024;-/// The decision a fall-through run produces: the selected model plus a human-readable reason.pub struct FallThroughDecision {/// Target selected by the classifier cascade.@@ -112,7 +106,7 @@ pub struct FallThrough<S = ()> {classifiers: Vec<Arc<dyn Classifier<S>>>,targets: LlmTargetSet,session_states: Option<SessionStates<S>>,- session_evictions: Mutex<HashMap<String, HashSet<String>>>,+ session_evictions: SessionEvictions,}impl FallThrough<()> {@@ -125,7 +119,7 @@ impl FallThrough<()> {classifiers: Vec::new(),targets,session_states: None,- session_evictions: Mutex::new(HashMap::new()),+ session_evictions: SessionEvictions::default(),}}}@@ -143,7 +137,7 @@ whereclassifiers: Vec::new(),targets,session_states: Some(Mutex::new(HashMap::new())),- session_evictions: Mutex::new(HashMap::new()),+ session_evictions: SessionEvictions::default(),}}@@ -182,14 +176,12 @@ wherelet mut request = request;let session = session_id(&request);let mut ctx = ctx;- for target in self.evicted_in_session(session.as_deref()) {- // Never seed the pool empty: a later turn may be small enough to serve, and the- // caller should get the upstream's answer rather than a routing error.- if self.eligible_targets(&ctx) <= 1 {- break;- }- ctx.exclude_target(target);- }+ exclude_evicted(+ &mut ctx,+ &self.targets,+ &self.session_evictions,+ session.as_deref(),+ );let session_state = self.session_state(&request);let (target, decision, served) = match session_state {Some(state) => {@@ -210,93 +202,22 @@ wherematch served {Some(response) => Ok(response),None => {- self.call_with_overflow_fallback(+ call_llm_with_overflow_fallback(ctx,&driver,+ &self.targets,target,decision,request,session.as_deref(),+ &self.session_evictions,+ |from, to| self.fallback_decision(from, to),).await}}}- fn eligible_targets(&self, ctx: &Context) -> usize {- self.targets- .targets()- .iter()- .filter(|t| !ctx.is_excluded(&t.semantic_name))- .count()- }-- fn evicted_in_session(&self, session: Option<&str>) -> Vec<String> {- let Some(session) = session else {- return Vec::new();- };- self.session_evictions- .lock()- .get(session)- .map(|targets| targets.iter().cloned().collect())- .unwrap_or_default()- }-- fn record_eviction(&self, session: Option<&str>, target: &str) {- let Some(session) = session else { return };- let mut sessions = self.session_evictions.lock();- if sessions.len() >= MAX_EVICTION_SESSIONS && !sessions.contains_key(session) {- if let Some(oldest) = sessions.keys().next().cloned() {- sessions.remove(&oldest);- }- }- sessions- .entry(session.to_string())- .or_default()- .insert(target.to_string());- }-- /// Calls `target`, falling back to the next eligible target whenever one overflows its- /// context window, until a call succeeds or every target has been tried.- ///- /// Routing is deliberately not re-run: the fallback replaces the target in place so the- /// processor chain and the retained session state still see exactly one turn.- async fn call_with_overflow_fallback(- &self,- mut ctx: Context,- driver: &Driver,- mut target: LlmTarget,- mut decision: Arc<dyn Decision>,- request: Request,- session: Option<&str>,- ) -> Result<Response> {- loop {- let result = driver- .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())- .await;- let Err(error) = result else { return result };- let LibsyError::ClientCall {- target: failed,- source: LlmClientError::ContextWindowExceeded { .. },- } = &error- else {- return Err(error);- };- // A target already excluded means the pool is spent; surface the client error- // so the caller still sees a context overflow rather than an internal failure.- if !ctx.exclude_target(failed) {- return Err(error);- }- self.record_eviction(session, failed);- let Ok(next) = self.targets.resolve_target(&target.semantic_name, &ctx) else {- return Err(error);- };- decision = self.fallback_decision(&target, &next);- target = next;- driver.info(ctx.clone(), decision.clone()).await?;- }- }-/// The decision published when an overflow sends the request to a different target.fn fallback_decision(&self, from: &LlmTarget, to: &LlmTarget) -> Arc<dyn Decision> {Arc::new(FallThroughDecision {@@ -438,7 +359,7 @@ mod tests {completion_text, text_request, text_response, LlmRequest, Message, Metadata, Role,};- use crate::{LlmResponse, LlmTarget, RoutedLlmClient};+ use crate::{LlmClientError, LlmResponse, LlmTarget, RoutedLlmClient};#[derive(Debug, thiserror::Error)]#[error("{0}")]diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex c99c1802f..84eb41d55 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -6,6 +6,7 @@//! calls and publishes [`Decision`]s over. See the crate root for the narrative model.use std::{+ collections::{HashMap, HashSet},pin::Pin,sync::Arc,time::{Duration, Instant},@@ -410,6 +411,123 @@ impl LlmTargetSet {}}+/// Matches the cap the Python stack uses. Dropping a live session's entry costs one+/// rediscovered overflow, so the victim choice does not need to be exact.+const MAX_EVICTION_SESSIONS: usize = 1_024;++/// Per-session record of the targets that overflowed their context window.+///+/// A conversation only grows, so a target that could not fit one turn will not fit a+/// later one; remembering it lets the next turn skip a call certain to fail. Requests+/// without a session id are not tracked — there is nothing to remember them by.+#[derive(Default)]+pub(crate) struct SessionEvictions {+ sessions: Mutex<HashMap<String, HashSet<String>>>,+}++impl SessionEvictions {+ /// The targets `session` has already overflowed; empty for an untracked request.+ fn evicted_in(&self, session: Option<&str>) -> Vec<String> {+ let Some(session) = session else {+ return Vec::new();+ };+ self.sessions+ .lock()+ .get(session)+ .map(|targets| targets.iter().cloned().collect())+ .unwrap_or_default()+ }++ /// Remembers that `target` overflowed in `session`, tracking at most+ /// [`MAX_EVICTION_SESSIONS`] sessions.+ fn record(&self, session: Option<&str>, target: &str) {+ let Some(session) = session else { return };+ let mut sessions = self.sessions.lock();+ if sessions.len() >= MAX_EVICTION_SESSIONS && !sessions.contains_key(session) {+ if let Some(oldest) = sessions.keys().next().cloned() {+ sessions.remove(&oldest);+ }+ }+ sessions+ .entry(session.to_string())+ .or_default()+ .insert(target.to_string());+ }+}++/// How many of `targets` this request is still allowed to reach.+fn eligible_targets(targets: &LlmTargetSet, ctx: &Context) -> usize {+ targets+ .targets()+ .iter()+ .filter(|t| !ctx.is_excluded(&t.semantic_name))+ .count()+}++/// Bars the targets `session` has already overflowed from this request, so routing does+/// not select one that is certain to fail again.+pub(crate) fn exclude_evicted(+ ctx: &mut Context,+ targets: &LlmTargetSet,+ evictions: &SessionEvictions,+ session: Option<&str>,+) {+ for target in evictions.evicted_in(session) {+ // Never seed the pool empty: a later turn may be small enough to serve, and the+ // caller should get the upstream's answer rather than a routing error.+ if eligible_targets(targets, ctx) <= 1 {+ break;+ }+ ctx.exclude_target(target);+ }+}++/// Calls `target`, falling back to the next eligible target in `targets` whenever one+/// overflows its context window, until a call succeeds or every target has been tried.+///+/// Routing is deliberately not re-run: the fallback replaces the target in place, so the+/// caller's request-side work and retained state still see exactly one turn.+/// `fallback_decision` builds the [`Decision`] published for a `from -> to` hop, and each+/// overflow is recorded against `session` so later turns skip that target outright.+#[allow(clippy::too_many_arguments)]+pub(crate) async fn call_llm_with_overflow_fallback(+ mut ctx: Context,+ driver: &Driver,+ targets: &LlmTargetSet,+ mut target: LlmTarget,+ mut decision: Arc<dyn Decision>,+ request: Request,+ session: Option<&str>,+ evictions: &SessionEvictions,+ fallback_decision: impl Fn(&LlmTarget, &LlmTarget) -> Arc<dyn Decision>,+) -> Result<Response> {+ loop {+ let result = driver+ .call_llm_target(ctx.clone(), &target, request.clone(), decision.clone())+ .await;+ let Err(error) = result else { return result };+ let LibsyError::ClientCall {+ target: failed,+ source: LlmClientError::ContextWindowExceeded { .. },+ } = &error+ else {+ return Err(error);+ };+ // A target already excluded means the pool is spent; surface the client error+ // so the caller still sees a context overflow rather than an internal failure.+ if !ctx.exclude_target(failed) {+ return Err(error);+ }+ evictions.record(session, failed);+ let Ok(next) = targets.resolve_target(&target.semantic_name, &ctx) else {+ return Err(error);+ };+ decision = fallback_decision(&target, &next);+ target = next;+ driver.info(ctx.clone(), decision.clone()).await?;+ }+}+/// An optimization strategy. Implement [`create_run_task`](Self::create_run_task);/// callers drive it with the provided [`run`](Self::run) (serve calls, get the answer)/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).