Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(libsy): generalize fall-through state (#158)

nachiketb-nvidia61f41ffd4f91
diff --git a/crates/libsy/README.md b/crates/libsy/README.md
index 9a0a6821f..2d1cc32bf 100644
--- a/crates/libsy/README.md
+++ b/crates/libsy/README.md
@@ -11,7 +11,7 @@ so it drops into a proxy, gateway, or agent runtime.
Build a target set, pick an algorithm, run a request:
```rust
-use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request, SharedState};
+use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request, State};
use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_protocol::{completion_text, text_request};
use std::sync::Arc;
@@ -20,11 +20,13 @@ use std::sync::Arc;
let client = Arc::new(MyClient { /* .. */ }) as Arc<dyn RoutedLlmClient>;
let target = |name: &str| LlmTarget { semantic_name: name.into(), llm_client: Some(client.clone()) };
-let targets = LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]);
-let judge_target = targets.get_target("classifier")?;
-let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(
- FallThrough::new(targets).with_classifier(Arc::new(LlmTaskClassifier::new(
- judge_target, "weak", "strong",
+let classifier = target("classifier");
+let strong = target("strong");
+let weak = target("weak");
+let targets = LlmTargetSet::new(vec![strong.clone(), weak.clone()]);
+let algo: Arc<dyn Algorithm> = Arc::new(
+ FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(LlmTaskClassifier::new(
+ classifier, weak, strong, 0.5,
)?)),
);
@@ -114,8 +116,7 @@ the id it calls — they can differ (`"strong"` → `"openai/gpt-4o"`) or coinci
## Running a request
-Hold the algorithm as `Arc<dyn Algorithm>` (or `Arc<dyn Algorithm<SharedState>>` for
-`FallThrough`) and choose one of two entry points:
+Hold the algorithm as `Arc<dyn Algorithm>` and choose one of two entry points:
```rust
// run: libsy drives the request to completion, serving each call with the target's
@@ -225,8 +226,9 @@ Compose a classifier into `FallThrough`; the fall-through algorithm calls the ju
applies its policy, and then invokes the selected target:
```rust
-let classifier = LlmTaskClassifier::new(judge_target, "weak", "strong")?;
-let router = FallThrough::new(targets).with_classifier(Arc::new(classifier));
+let classifier = LlmTaskClassifier::new(judge_target, weak_target, strong_target, 0.5)?;
+let router =
+ FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(classifier));
```
## Errors
diff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rs
index f9ff0f44d..0056ac02b 100644
--- a/crates/libsy/examples/research_agent.rs
+++ b/crates/libsy/examples/research_agent.rs
@@ -17,7 +17,7 @@ use async_trait::async_trait;
use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_libsy::{
Algorithm, Context, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request,
- Response, Result, RoutedLlmClient, SharedState,
+ Response, Result, RoutedLlmClient, State,
};
use switchyard_protocol::{completion_text, text_request, text_response};
@@ -64,7 +64,7 @@ fn targets() -> LlmTargetSet {
}
struct ResearchAgent {
- algo: Arc<dyn Algorithm<SharedState>>,
+ algo: Arc<dyn Algorithm>,
}
impl ResearchAgent {
@@ -102,10 +102,10 @@ async fn main() -> Result<()> {
let classifier = target_set.get_target(CLASSIFIER)?;
let weak = target_set.get_target(WEAK)?;
let strong = target_set.get_target(STRONG)?;
- let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(
- FallThrough::new(target_set).with_classifier(Arc::new(LlmTaskClassifier::new(
- classifier, weak, strong, THRESHOLD,
- )?)),
+ let algo: Arc<dyn Algorithm> = Arc::new(
+ FallThrough::<State>::new_with_state(target_set).with_classifier(Arc::new(
+ LlmTaskClassifier::new(classifier, weak, strong, THRESHOLD)?,
+ )),
);
let agent = ResearchAgent { algo };
diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rs
index f0fc11ead..475cf468f 100644
--- a/crates/libsy/examples/research_agent_core.rs
+++ b/crates/libsy/examples/research_agent_core.rs
@@ -14,7 +14,7 @@ use std::sync::Arc;
use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_libsy::{
Algorithm, Context, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request,
- Response, Result, SharedState, Step,
+ Response, Result, State, Step,
};
use switchyard_protocol::{completion_text, text_request, text_response};
use tokio_stream::StreamExt;
@@ -51,7 +51,7 @@ fn targets() -> LlmTargetSet {
}
struct ResearchAgent {
- algo: Arc<dyn Algorithm<SharedState>>,
+ algo: Arc<dyn Algorithm>,
}
impl ResearchAgent {
@@ -110,10 +110,10 @@ async fn main() -> Result<()> {
let classifier = target_set.get_target(CLASSIFIER)?;
let weak = target_set.get_target(WEAK)?;
let strong = target_set.get_target(STRONG)?;
- let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(
- FallThrough::new(target_set).with_classifier(Arc::new(LlmTaskClassifier::new(
- classifier, weak, strong, THRESHOLD,
- )?)),
+ let algo: Arc<dyn Algorithm> = Arc::new(
+ FallThrough::<State>::new_with_state(target_set).with_classifier(Arc::new(
+ LlmTaskClassifier::new(classifier, weak, strong, THRESHOLD)?,
+ )),
);
let mut agent = ResearchAgent { algo };
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index 422a6865a..1d64e936f 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -6,7 +6,7 @@
//! Reach for them by name — `use switchyard_libsy::algorithms::Random` — rather than through the
//! per-algorithm submodules.
-pub mod fall_through;
+mod fall_through;
pub mod llm_class;
pub mod noop;
pub mod passthrough;
@@ -16,7 +16,10 @@ pub use fall_through::{FallThrough, FallThroughDecision};
pub use llm_class::LlmTaskClassifier;
pub use noop::{Noop, NoopDecision};
pub use passthrough::{Passthrough, PassthroughDecision};
-pub use rand::{Random, RandomDecision};
+pub use rand::{Random, RandomClassifier, RandomDecision};
pub use util::{AffinityRouter, SubagentOverride};
pub mod util;
+
+#[cfg(test)]
+mod subagent_affinity_tests;
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index b95f7b578..0152af477 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -1,40 +1,44 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Fall-through classifier routing: a stateful [`Algorithm`] that routes each turn
+//! Fall-through classifier routing: a composable [`Algorithm`] that routes each turn
//! through a processor chain and a classifier cascade.
//!
-//! Each turn: request-side [`Processor`]s fold facts into the session [`State`](crate::State); the
+//! Each turn: request-side [`Processor`]s fold facts into the composition's state; the
//! [`Classifier`] cascade is consulted in order and the first to score decides the target
//! (its `argmax`); the [`Decision`] is published and then replayed to the processors so
//! stateful ones (latch, affinity) can bind it.
//!
-//! [`FallThrough`] itself is stateless — one shared instance serves every session. The
-//! per-session [`State`](crate::State) rides in the threaded [`Context<SharedState>`] the caller passes into
-//! each turn, so routing can depend on accumulated history rather than the current request
-//! alone. The state is held under a lock, so concurrent turns of the same session serialize
-//! on it.
+//! The default `FallThrough<()>` has no state. Stateful compositions share one private state
+//! value across turns with the same session ID. Requests without a session ID use unretained
+//! per-run state.
-use std::sync::Arc;
+use std::{collections::HashMap, sync::Arc};
use async_trait::async_trait;
+use parking_lot::Mutex;
+use tokio::sync::Mutex as AsyncMutex;
-use crate::core::{Classifier, Event, Processor, Score, SharedState};
+use crate::core::{Classifier, Event, Processor, Score};
use crate::{
Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,
RoutedLlmClient,
};
+type SessionStates<S> = Mutex<HashMap<String, Arc<AsyncMutex<S>>>>;
+
/// The decision a fall-through run produces: the selected model plus a human-readable reason.
pub struct FallThroughDecision {
- model: String,
- reason: String,
+ /// Target selected by the classifier cascade.
+ pub selected_model: String,
+ /// Human-readable explanation of the selection.
+ pub reasoning: String,
tier: Option<&'static str>,
}
impl Decision for FallThroughDecision {
fn selected_model(&self) -> &str {
- &self.model
+ &self.selected_model
}
fn routing_tier(&self) -> Option<&str> {
@@ -42,7 +46,7 @@ impl Decision for FallThroughDecision {
}
fn reasoning(&self) -> Option<&str> {
- Some(&self.reason)
+ Some(&self.reasoning)
}
fn as_any(&self) -> &dyn std::any::Any {
@@ -52,36 +56,69 @@ impl Decision for FallThroughDecision {
/// Processor chain → classifier cascade → routed model call. See the [module docs](self).
///
-/// Stateless: the per-session [`State`](crate::State) lives in the threaded [`Context<SharedState>`], so one
-/// shared instance serves every session (see the module docs).
-pub struct FallThrough {
- processors: Vec<Arc<dyn Processor>>,
- classifiers: Vec<Arc<dyn Classifier>>,
+/// The generic state type is shared by every processor and classifier in the composition.
+pub struct FallThrough<S = ()> {
+ name: String,
+ decision_reason: fn(&str, &Score) -> String,
+ processors: Vec<Arc<dyn Processor<S>>>,
+ classifiers: Vec<Arc<dyn Classifier<S>>>,
targets: LlmTargetSet,
+ session_states: Option<SessionStates<S>>,
}
-impl FallThrough {
- /// Creates an empty router over `targets`; add components with the `with_*` builders.
+impl FallThrough<()> {
+ /// Creates an empty stateless router.
pub fn new(targets: LlmTargetSet) -> Self {
Self {
+ name: "fall_through".to_string(),
+ decision_reason: default_decision_reason,
+ processors: Vec::new(),
+ classifiers: Vec::new(),
+ targets,
+ session_states: None,
+ }
+ }
+}
+
+impl<S> FallThrough<S>
+where
+ S: Default + Send + 'static,
+{
+ /// Creates a router that retains one private `S` per session.
+ pub fn new_with_state(targets: LlmTargetSet) -> Self {
+ Self {
+ name: "fall_through".to_string(),
+ decision_reason: default_decision_reason,
processors: Vec::new(),
classifiers: Vec::new(),
targets,
+ session_states: Some(Mutex::new(HashMap::new())),
}
}
+ /// Sets the stable, low-cardinality telemetry name for this composition.
+ pub fn with_name(mut self, name: impl Into<String>) -> Self {
+ self.name = name.into();
+ self
+ }
+
+ /// Sets the decision reasoning for an algorithm assembled from this cascade.
+ pub(crate) fn with_decision_reason(mut self, reason: fn(&str, &Score) -> String) -> Self {
+ self.decision_reason = reason;
+ self
+ }
+
/// Appends a processor to the head-of-request chain.
- pub fn with_processor(mut self, processor: Arc<dyn Processor>) -> Self {
+ pub fn with_processor(mut self, processor: Arc<dyn Processor<S>>) -> Self {
self.processors.push(processor);
self
}
/// Appends a classifier to the cascade.
- pub fn with_classifier(mut self, classifier: Arc<dyn Classifier>) -> Self {
+ pub fn with_classifier(mut self, classifier: Arc<dyn Classifier<S>>) -> Self {
self.classifiers.push(classifier);
self
}
-
/// Registers one dual-role component in *both* the processor chain and the classifier
/// cascade.
///
@@ -90,41 +127,67 @@ impl FallThrough {
/// shares that state through the instance, so both roles must be the same `Arc`.
/// Registering the two separately is easy to half-wire: omit the processor and the
/// classifier silently never sees an assignment. This registers both at once.
- pub fn with_component<T: Processor + Classifier + 'static>(self, component: Arc<T>) -> Self {
+ pub fn with_component<T>(self, component: Arc<T>) -> Self
+ where
+ T: Processor<S> + Classifier<S> + 'static,
+ {
self.with_processor(component.clone())
.with_classifier(component)
}
-}
-#[async_trait]
-impl Algorithm<SharedState> for FallThrough {
- fn name(&self) -> &str {
- "fall_through"
- }
- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
- self.targets.count_tokens_client()
- }
-
- async fn create_run_task(
- self: Arc<Self>,
- ctx: Context<SharedState>,
+ /// Executes the processor/classifier/target-call sequence for wrappers and the trait entrypoint.
+ pub(crate) async fn execute(
+ &self,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- // Hold the session state (carried in the threaded context) for the whole
- // request→decision fold, so a turn's fact accumulation is atomic and concurrent
- // turns of the same session serialize on it.
- let mut state = ctx.state.lock().await;
-
// The request is threaded mutably through the whole fold: any component may rewrite
- // it, later components see the rewrite, and the final value is what reaches the model.
+ // it, later components see the rewrite, and the final value reaches the model.
let mut request = request;
+ let session_state = self.session_state(&request);
+ let (target, decision) = match session_state {
+ Some(state) => {
+ let mut state = state.lock().await;
+ self.route(&mut state, &ctx, &driver, &mut request).await?
+ }
+ None => {
+ let mut state = S::default();
+ self.route(&mut state, &ctx, &driver, &mut request).await?
+ }
+ };
+
+ driver
+ .call_llm_target(ctx, &target, request, decision)
+ .await
+ }
- // 1. Processor chain accumulates request-side facts into the session State.
+ /// Returns this request's retained state without holding the registry lock.
+ fn session_state(&self, request: &Request) -> Option<Arc<AsyncMutex<S>>> {
+ let states = self.session_states.as_ref()?;
+ let session_id = request
+ .metadata
+ .as_ref()?
+ .session_id
+ .as_deref()
+ .filter(|session_id| !session_id.is_empty())?;
+ let mut states = states.lock();
+ let state = states
+ .entry(session_id.to_string())
+ .or_insert_with(|| Arc::new(AsyncMutex::new(S::default())));
+ Some(Arc::clone(state))
+ }
+
+ async fn route(
+ &self,
+ state: &mut S,
+ ctx: &Context,
+ driver: &Driver,
+ request: &mut Request,
+ ) -> Result<(crate::LlmTarget, Arc<dyn Decision>)> {
+ // 1. Processor chain accumulates request-side facts into the composition's state.
for processor in &self.processors {
- processor
- .process(&mut state, Event::Request(&mut request))
- .await?;
+ processor.process(state, Event::Request(request)).await?;
}
// 2. Fall through the cascade: the first classifier to score decides (argmax). The
@@ -132,9 +195,7 @@ impl Algorithm<SharedState> for FallThrough {
let mut maybe_score: Option<Score> = None;
let mut maybe_tier: Option<&'static str> = None;
for classifier in &self.classifiers {
- let scores = classifier
- .score(&mut state, &mut request, Some(&driver))
- .await?;
+ let scores = classifier.score(state, request, Some(driver)).await?;
maybe_score = scores.argmax(false)?;
if let Some(s) = maybe_score.as_ref() {
maybe_tier = classifier.routing_tier(&s.target);
@@ -150,33 +211,63 @@ impl Algorithm<SharedState> for FallThrough {
// 3. Resolve the target and publish the decision.
let target = self.targets.get_target(&score.target)?;
let decision: Arc<dyn Decision> = Arc::new(FallThroughDecision {
- model: score.target.clone(),
- reason: format!(
- "fall-through selected {} (confidence {:.3})",
- score.target, score.confidence
- ),
+ selected_model: score.target.clone(),
+ reasoning: (self.decision_reason)(&self.name, &score),
tier: maybe_tier,
});
- driver.info(ctx.without_state(), decision.clone()).await?;
+ driver.info(ctx.clone(), decision.clone()).await?;
- // 4. Replay the decision to the processors so stateful ones (latch / affinity) bind
- // it into the session State.
+ // 4. Replay the decision to the processors so stateful ones can bind it.
for processor in &self.processors {
processor
- .process(&mut state, Event::Decision(decision.as_ref()))
+ .process(
+ state,
+ Event::Decision {
+ request,
+ decision: decision.as_ref(),
+ },
+ )
.await?;
}
- driver
- .call_llm_target(ctx.without_state(), &target, request, decision)
- .await
+ Ok((target, decision))
+ }
+}
+
+fn default_decision_reason(_name: &str, winner: &Score) -> String {
+ format!(
+ "fall-through selected {} (confidence {:.3})",
+ winner.target, winner.confidence
+ )
+}
+
+#[async_trait]
+impl<S> Algorithm for FallThrough<S>
+where
+ S: Default + Send + 'static,
+{
+ fn name(&self) -> &str {
+ &self.name
+ }
+
+ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
+ self.targets.count_tokens_client()
+ }
+
+ async fn create_run_task(
+ self: Arc<Self>,
+ ctx: Context,
+ driver: Driver,
+ request: Request,
+ ) -> Result<Response> {
+ self.execute(ctx, driver, request).await
}
}
#[cfg(test)]
mod tests {
use super::*;
- use crate::core::{Classification, State};
+ use crate::core::Classification;
use switchyard_protocol::{
completion_text, text_response, LlmRequest, Message, Metadata, Role,
@@ -258,7 +349,7 @@ mod tests {
impl Classifier for FixedClassifier {
async fn score(
&self,
- _state: &mut State,
+ _state: &mut (),
_request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
@@ -300,13 +391,15 @@ mod tests {
}
}
- /// Drives a shared router through one turn on the given session context, returning the
- /// completion text + trace. Reuse the same `ctx` across calls to model a session.
- async fn run_turn(
- router: &Arc<FallThrough>,
- ctx: Context<SharedState>,
- ) -> Result<(String, Vec<Arc<dyn Decision>>)> {
- let (trace, response) = router.clone().run(ctx, request()).await?;
+ /// Drives a shared router with one request, returning the completion text + trace.
+ async fn run_request<S>(
+ router: &Arc<FallThrough<S>>,
+ request: Request,
+ ) -> Result<(String, Vec<Arc<dyn Decision>>)>
+ where
+ S: Default + Send + 'static,
+ {
+ let (trace, response) = router.clone().run(Context::default(), request).await?;
let text = response
.llm_response
.into_agg()
@@ -316,16 +409,24 @@ mod tests {
Ok((text, trace))
}
- /// Drives a fresh router through one turn on a fresh session.
+ /// Drives a shared router through one turn in the default test session.
+ async fn run_turn<S>(router: &Arc<FallThrough<S>>) -> Result<(String, Vec<Arc<dyn Decision>>)>
+ where
+ S: Default + Send + 'static,
+ {
+ run_request(router, request()).await
+ }
+
+ /// Drives a fresh router through one turn.
async fn run(router: FallThrough) -> Result<(String, Vec<Arc<dyn Decision>>)> {
- run_turn(&Arc::new(router), Context::default()).await
+ run_turn(&Arc::new(router)).await
}
// --- tests -------------------------------------------------------------------------
#[tokio::test]
async fn argmax_picks_the_highest_confidence_target() -> Result<()> {
- let router = FallThrough::new(target_set(&["strong", "weak"]))
+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(fixed(vec![score("weak", 0.2), score("strong", 0.9)]));
let (model, trace) = run(router).await?;
assert_eq!(model, "strong");
@@ -337,7 +438,7 @@ mod tests {
#[tokio::test]
async fn falls_through_the_first_abstaining_classifier() -> Result<()> {
// First classifier abstains (empty); the second decides.
- let router = FallThrough::new(target_set(&["strong", "weak"]))
+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(fixed(vec![]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
let (model, _) = run(router).await?;
@@ -348,7 +449,7 @@ mod tests {
#[tokio::test]
async fn first_deciding_classifier_wins_the_cascade() -> Result<()> {
// The first classifier decides; the second is never consulted.
- let router = FallThrough::new(target_set(&["strong", "weak"]))
+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(fixed(vec![score("strong", 0.6)]))
.with_classifier(fixed(vec![score("weak", 1.0)]));
let (model, _) = run(router).await?;
@@ -359,7 +460,7 @@ mod tests {
#[tokio::test]
async fn all_abstaining_is_an_error() -> Result<()> {
let router =
- FallThrough::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![]));
+ FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![]));
let error = run(router)
.await
.err()
@@ -381,7 +482,7 @@ mod tests {
impl Classifier for NeedsDriver {
async fn score(
&self,
- _state: &mut State,
+ _state: &mut (),
_request: &mut Request,
driver: Option<&Driver>,
) -> Result<Classification> {
@@ -392,7 +493,7 @@ mod tests {
}
}
- let router = FallThrough::new(target_set(&["strong", "weak"]))
+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_classifier(Arc::new(NeedsDriver));
let (model, _) = run(router).await?;
assert_eq!(model, "strong");
@@ -408,10 +509,10 @@ mod tests {
#[async_trait]
impl Processor for RecordingProcessor {
- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
let kind = match event {
Event::Request(_) => "request",
- Event::Decision(_) => "decision",
+ Event::Decision { .. } => "decision",
_ => "other",
};
self.0.lock().push(kind);
@@ -420,7 +521,7 @@ mod tests {
}
let seen = Arc::new(Mutex::new(Vec::new()));
- let router = FallThrough::new(target_set(&["strong", "weak"]))
+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"]))
.with_processor(Arc::new(RecordingProcessor(seen.clone())))
.with_classifier(fixed(vec![score("strong", 1.0)]));
run(router).await?;
@@ -438,7 +539,7 @@ mod tests {
#[async_trait]
impl Processor for Appender {
- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
if let Event::Request(request) = event {
request
.llm_request
@@ -457,7 +558,7 @@ mod tests {
impl Classifier for TrailClassifier {
async fn score(
&self,
- _state: &mut State,
+ _state: &mut (),
request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
@@ -486,7 +587,7 @@ mod tests {
.with_processor(Arc::new(Appender("second")))
.with_classifier(Arc::new(TrailClassifier(seen_by_classifier.clone())));
- run_turn(&Arc::new(router), Context::default()).await?;
+ run_turn(&Arc::new(router)).await?;
// The classifier saw both processors' edits, in chain order, on top of the original.
assert_eq!(*seen_by_classifier.lock(), vec!["hi", "first", "second"]);
@@ -507,58 +608,72 @@ mod tests {
}
#[tokio::test]
- async fn state_persists_across_turns() -> Result<()> {
+ async fn state_is_shared_within_a_session_and_isolated_between_sessions() -> Result<()> {
+ #[derive(Default)]
+ struct TurnState {
+ count: u32,
+ }
+
// Increments the session turn count on every request.
struct CountingProcessor;
#[async_trait]
- impl Processor for CountingProcessor {
- async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
+ impl Processor<TurnState> for CountingProcessor {
+ async fn process(&self, state: &mut TurnState, event: Event<'_>) -> Result<()> {
if let Event::Request(_) = event {
- state.turn_count += 1;
+ state.count += 1;
}
Ok(())
}
}
- // Routes to "weak" until the session has accumulated >= 2 turns, then "strong" —
- // its decision depends only on state carried over from earlier turns.
+ // Routes weak on a session's first turn and strong on later turns.
struct ThresholdClassifier;
#[async_trait]
- impl Classifier for ThresholdClassifier {
+ impl Classifier<TurnState> for ThresholdClassifier {
async fn score(
&self,
- state: &mut State,
+ state: &mut TurnState,
_request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
- let target = if state.turn_count >= 2 {
- "strong"
- } else {
- "weak"
- };
+ let target = if state.count >= 2 { "strong" } else { "weak" };
Ok(Classification::Scores(vec![score(target, 1.0)]))
}
}
- // One shared, stateless router; the session lives in the context we thread through.
let router = Arc::new(
- FallThrough::new(target_set(&["strong", "weak"]))
+ FallThrough::<TurnState>::new_with_state(target_set(&["strong", "weak"]))
.with_processor(Arc::new(CountingProcessor))
.with_classifier(Arc::new(ThresholdClassifier)),
);
- // One session context, reused across three turns. The turn counter accumulates in
- // its `State`, so the classifier crosses its threshold on turn 2.
- let ctx = Context::<SharedState>::default();
- let (turn1, _) = run_turn(&router, ctx.clone()).await?;
- let (turn2, _) = run_turn(&router, ctx.clone()).await?;
- let (turn3, _) = run_turn(&router, ctx.clone()).await?;
-
- assert_eq!(turn1, "weak"); // count 1 — below threshold
- assert_eq!(turn2, "strong"); // count 2 — state carried over from turn 1
- assert_eq!(turn3, "strong"); // count 3 — still above threshold
+ let (turn1, _) = run_turn(&router).await?;
+ let (turn2, _) = run_turn(&router).await?;
+ let (second_session, _) = run_request(
+ &router,
+ Request {
+ metadata: Some(Metadata {
+ session_id: Some("session-2".to_string()),
+ ..Metadata::default()
+ }),
+ ..request()
+ },
+ )
+ .await?;
+ let anonymous = Request {
+ metadata: None,
+ ..request()
+ };
+ let (anonymous1, _) = run_request(&router, anonymous.clone()).await?;
+ let (anonymous2, _) = run_request(&router, anonymous).await?;
+
+ assert_eq!(turn1, "weak");
+ assert_eq!(turn2, "strong");
+ assert_eq!(second_session, "weak");
+ assert_eq!(anonymous1, "weak");
+ assert_eq!(anonymous2, "weak");
Ok(())
}
}
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index eb5061bf3..a3dc63c57 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -230,7 +230,7 @@ impl LlmTaskClassifier {
}
#[async_trait]
-impl Classifier for LlmTaskClassifier {
+impl Classifier<State> for LlmTaskClassifier {
fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {
if self.efficient_target == self.capable_target {
None
@@ -262,9 +262,7 @@ mod tests {
use super::*;
use switchyard_protocol::{completion_text, text_response, LlmClientError};
- use crate::{
- Algorithm, Context, LlmResponse, LlmTargetSet, Response, RoutedLlmClient, SharedState,
- };
+ use crate::{Algorithm, Context, LlmResponse, LlmTargetSet, Response, RoutedLlmClient};
const TEST_THRESHOLD: f64 = 0.5;
@@ -354,7 +352,7 @@ mod tests {
}
}
- fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<super::super::FallThrough>> {
+ fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<super::super::FallThrough<State>>> {
let target = |name: &str| LlmTarget {
semantic_name: name.to_string(),
llm_client: Some(client.clone()),
@@ -363,7 +361,7 @@ mod tests {
let efficient = targets.get_target("efficient")?;
let capable = targets.get_target("capable")?;
Ok(Arc::new(
- super::super::FallThrough::new(targets).with_classifier(Arc::new(
+ super::super::FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(
LlmTaskClassifier::new(target("judge"), efficient, capable, TEST_THRESHOLD)?,
)),
))
@@ -384,9 +382,7 @@ mod tests {
async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {
let router = router(Arc::new(UnreachableJudgeClient))?;
- let (trace, response) = router
- .run(Context::<SharedState>::default(), classify_request())
- .await?;
+ let (trace, response) = router.run(Context::default(), classify_request()).await?;
assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));
assert_eq!(
@@ -402,14 +398,8 @@ mod tests {
let router = router(client.clone())?;
let request = classify_request;
- router
- .clone()
- .run(Context::<SharedState>::default(), request())
- .await?;
- router
- .clone()
- .run(Context::<SharedState>::default(), request())
- .await?;
+ router.clone().run(Context::default(), request()).await?;
+ router.clone().run(Context::default(), request()).await?;
assert_eq!(
client.calls(),
diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rs
index 9b86746bd..389a1e72a 100644
--- a/crates/libsy/src/algorithms/noop.rs
+++ b/crates/libsy/src/algorithms/noop.rs
@@ -34,17 +34,14 @@ impl Decision for NoopDecision {
}
#[async_trait::async_trait]
-impl<S> Algorithm<S> for Noop
-where
- S: Clone + Send + Sync + 'static,
-{
+impl Algorithm for Noop {
fn name(&self) -> &str {
"noop"
}
async fn create_run_task(
self: Arc<Self>,
- ctx: Context<S>,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
@@ -55,7 +52,7 @@ where
let decision: Arc<dyn Decision> = Arc::new(NoopDecision {
model: model.clone(),
});
- driver.info(ctx.without_state(), decision.clone()).await?;
+ driver.info(ctx, decision.clone()).await?;
let llm_response = LlmResponse::Agg(AggLlmResponse {
id: Some("switchyard-noop".to_string()),
diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs
index 4037cde68..35022ecd6 100644
--- a/crates/libsy/src/algorithms/passthrough.rs
+++ b/crates/libsy/src/algorithms/passthrough.rs
@@ -40,10 +40,7 @@ impl Decision for PassthroughDecision {
}
#[async_trait::async_trait]
-impl<S> Algorithm<S> for Passthrough
-where
- S: Clone + Send + Sync + 'static,
-{
+impl Algorithm for Passthrough {
fn name(&self) -> &str {
"passthrough"
}
@@ -58,14 +55,13 @@ where
async fn create_run_task(
self: Arc<Self>,
- ctx: Context<S>,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
let decision: Arc<dyn Decision> = Arc::new(PassthroughDecision {
model_id: self.target.semantic_name.clone(),
});
- let ctx = ctx.without_state();
driver.info(ctx.clone(), decision.clone()).await?;
driver
.call_llm_target(ctx, &self.target, request, decision)
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 40f1abcec..60471a125 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -1,55 +1,38 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Random router built on the [`Algorithm`] interfaces.
+//! Random routing as a stateless [`FallThrough`] composition.
//!
-//! Selects one target from the set at random and calls it. Selection is uniform
-//! by default and can use relative weights and a reproducible seed.
+//! [`RandomClassifier`] selects one target; [`FallThrough`] owns the common
+//! processor/classifier/target-call orchestration.
use std::collections::BTreeSet;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
use async_trait::async_trait;
+use parking_lot::Mutex;
use rand::distributions::{Distribution, WeightedIndex};
use rand::rngs::StdRng;
use rand::SeedableRng;
+use crate::algorithms::fall_through::{FallThrough, FallThroughDecision};
use crate::{
- Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,
- Result, RoutedLlmClient,
+ Algorithm, Classification, Classifier, Context, Driver, LibsyError, LlmTargetSet, Request,
+ Response, Result, RoutedLlmClient, Score,
};
-/// Decision produced by [`Random`]: which target was chosen and why.
-pub struct RandomDecision {
- /// The randomly selected target/model.
- pub selected_model: String,
- /// Human-readable explanation of the choice.
- pub reasoning: String,
-}
-
-impl Decision for RandomDecision {
- fn selected_model(&self) -> &str {
- &self.selected_model
- }
-
- fn reasoning(&self) -> Option<&str> {
- Some(&self.reasoning)
- }
+/// Compatibility name for the decision produced by [`Random`].
+pub type RandomDecision = FallThroughDecision;
- fn as_any(&self) -> &dyn std::any::Any {
- self
- }
-}
-
-/// Random router over a target set.
-pub struct Random {
- target_set: LlmTargetSet,
+/// Stateless weighted classifier used by random fall-through routing.
+pub struct RandomClassifier {
+ targets: Vec<String>,
distribution: WeightedIndex<f64>,
rng: Mutex<StdRng>,
}
-impl Random {
- /// Creates a router over `target_set`.
+impl RandomClassifier {
+ /// Creates a classifier over ordered target names.
///
/// Missing weights default to one per target. Explicit weights are relative,
/// follow target order, and need not sum to one. Zero disables a target.
@@ -60,20 +43,12 @@ impl Random {
/// Returns an error when targets are empty or duplicated, or when explicit
/// weights have the wrong length, are negative or non-finite, or contain no
/// positive value.
- pub fn new(
- target_set: LlmTargetSet,
- weights: Option<Vec<f64>>,
- seed: Option<u64>,
- ) -> Result<Self> {
- let target_count = target_set.targets().len();
+ pub fn new(targets: Vec<String>, weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Self> {
+ let target_count = targets.len();
if target_count == 0 {
return Err(LibsyError::NoTargets);
}
- let unique_targets = target_set
- .targets()
- .iter()
- .map(|target| target.semantic_name.as_str())
- .collect::<BTreeSet<_>>();
+ let unique_targets = targets.iter().map(String::as_str).collect::<BTreeSet<_>>();
if unique_targets.len() != target_count {
return Err(LibsyError::AlgorithmError {
message: "random targets must be unique".to_string(),
@@ -107,19 +82,16 @@ impl Random {
None => StdRng::from_entropy(),
};
Ok(Self {
- target_set,
+ targets,
distribution,
rng: Mutex::new(rng),
})
}
- fn select_target(&self) -> Result<LlmTarget> {
- let targets = self.target_set.targets();
- let mut rng = self.rng.lock().map_err(|_| LibsyError::AlgorithmError {
- message: "random number generator lock was poisoned".to_string(),
- })?;
+ fn select_target(&self) -> String {
+ let mut rng = self.rng.lock();
let index = self.distribution.sample(&mut *rng);
- Ok(targets[index].clone())
+ self.targets[index].clone()
}
}
@@ -130,37 +102,74 @@ fn invalid_weights(message: String) -> LibsyError {
}
#[async_trait]
-impl<S> Algorithm<S> for Random
+impl<S> Classifier<S> for RandomClassifier
where
- S: Clone + Send + Sync + 'static,
+ S: Send + 'static,
{
+ async fn score(
+ &self,
+ _state: &mut S,
+ _request: &mut Request,
+ _driver: Option<&Driver>,
+ ) -> Result<Classification> {
+ Ok(Classification::Scores(vec![Score {
+ confidence: 1.0,
+ target: self.select_target(),
+ }]))
+ }
+}
+
+/// Random router implemented as a stateless fall-through composition.
+pub struct Random {
+ inner: FallThrough<()>,
+}
+
+impl Random {
+ /// Creates a router over `target_set`.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when targets or weights are invalid for [`RandomClassifier`].
+ pub fn new(
+ target_set: LlmTargetSet,
+ weights: Option<Vec<f64>>,
+ seed: Option<u64>,
+ ) -> Result<Self> {
+ let target_names = target_set
+ .targets()
+ .iter()
+ .map(|target| target.semantic_name.clone())
+ .collect();
+ let classifier = Arc::new(RandomClassifier::new(target_names, weights, seed)?);
+ let inner = FallThrough::<()>::new(target_set)
+ .with_name("random")
+ .with_decision_reason(random_decision_reason)
+ .with_classifier(classifier);
+ Ok(Self { inner })
+ }
+}
+
+fn random_decision_reason(_name: &str, winner: &Score) -> String {
+ format!("random routing selected target '{}'", winner.target)
+}
+
+#[async_trait]
+impl Algorithm for Random {
fn name(&self) -> &str {
"random"
}
fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {
- self.target_set.count_tokens_client()
+ self.inner.count_tokens_client()
}
async fn create_run_task(
self: Arc<Self>,
- ctx: Context<S>,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response> {
- let target = self.select_target()?;
-
- let selected = target.semantic_name.clone();
- let decision: Arc<dyn Decision> = Arc::new(RandomDecision {
- reasoning: format!("random routing selected target '{selected}'"),
- selected_model: selected,
- });
-
- let ctx = ctx.without_state();
- driver.info(ctx.clone(), Arc::clone(&decision)).await?;
- driver
- .call_llm_target(ctx, &target, request, decision)
- .await
+ self.inner.execute(ctx, driver, request).await
}
}
@@ -169,9 +178,10 @@ mod tests {
use super::*;
use std::collections::HashSet;
- use switchyard_protocol::{completion_text, text_request, text_response};
+ use switchyard_protocol::{completion_text, text_request, text_response, Metadata};
- use crate::{LlmResponse, LlmTarget, Request, RoutedLlmClient, Signals};
+ use crate::algorithms::AffinityRouter;
+ use crate::{Decision, LlmResponse, LlmTarget, Request, RoutedLlmClient, Signals};
/// Echoes the selected target so tests can inspect which target was called.
struct EchoClient;
@@ -199,8 +209,17 @@ mod tests {
}
}
- /// Builds a random router whose targets all share an echo client.
- fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {
+ fn request_for_session(session_id: &str) -> Request {
+ Request {
+ metadata: Some(Metadata {
+ session_id: Some(session_id.to_string()),
+ ..Metadata::default()
+ }),
+ ..request()
+ }
+ }
+
+ fn target_set(names: &[&str]) -> LlmTargetSet {
let targets = names
.iter()
.map(|name| LlmTarget {
@@ -208,7 +227,12 @@ mod tests {
llm_client: Some(Arc::new(EchoClient)),
})
.collect();
- Random::new(LlmTargetSet::new(targets), weights, seed)
+ LlmTargetSet::new(targets)
+ }
+
+ /// Builds a random router whose targets all share an echo client.
+ fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {
+ Random::new(target_set(names), weights, seed)
}
fn shared_algorithm(names: &[&str]) -> Result<Arc<dyn Algorithm>> {
@@ -322,6 +346,58 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn affinity_reuses_the_initial_random_selection() -> Result<()> {
+ let names = ["a/model", "b/model"];
+ let affinity = Arc::new(AffinityRouter::new());
+ let random = Arc::new(RandomClassifier::new(
+ names.iter().map(|name| (*name).to_string()).collect(),
+ None,
+ Some(42),
+ )?);
+ let algorithm: Arc<dyn Algorithm> = Arc::new(
+ FallThrough::<()>::new(target_set(&names))
+ .with_name("affinity_random")
+ .with_processor(affinity.clone())
+ .with_classifier(affinity.clone())
+ .with_classifier(random),
+ );
+
+ let (_, first) = algorithm
+ .clone()
+ .run(Context::default(), request_for_session("session-1"))
+ .await?;
+ let selected = first
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default();
+
+ let mut state = ();
+ let mut request = request_for_session("session-1");
+ let retained = affinity
+ .score(&mut state, &mut request, None)
+ .await?
+ .argmax(false)?;
+ assert_eq!(
+ retained.map(|score| score.target),
+ Some(selected.to_string())
+ );
+
+ let (_, second) = algorithm
+ .run(Context::default(), request_for_session("session-1"))
+ .await?;
+ assert_eq!(
+ second
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ selected
+ );
+ Ok(())
+ }
+
#[test]
fn rejects_invalid_weights() {
let cases = [
diff --git a/crates/libsy/tests/subagent_affinity.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
similarity index 73%
rename from crates/libsy/tests/subagent_affinity.rs
rename to crates/libsy/src/algorithms/subagent_affinity_tests.rs
index fca6403d7..ba7cd1d65 100644
--- a/crates/libsy/tests/subagent_affinity.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -1,22 +1,22 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-//! Integration tests for composing the sub-agent override with affinity routing.
+//! Tests for composing the sub-agent override with affinity routing.
//!
//! The two are independent classifiers in one [`FallThrough`] cascade: the override decides
//! *which* target delegated work belongs on, affinity decides *how long* that decision
-//! lives. These tests drive them through the crate's public API, so they also pin that the
-//! pieces needed to compose a cascade are actually exported.
+//! lives.
use std::collections::BTreeMap;
use std::sync::Arc;
use async_trait::async_trait;
-use switchyard_libsy::algorithms::{AffinityRouter, FallThrough, SubagentOverride};
-use switchyard_libsy::{
+use super::fall_through::FallThrough;
+use super::{AffinityRouter, SubagentOverride};
+use crate::{
Algorithm, Classification, Classifier, Context, Decision, Driver, LlmResponse, LlmTarget,
- LlmTargetSet, Metadata, Request, Response, Result, RoutedLlmClient, Score, SharedState, State,
+ LlmTargetSet, Metadata, Request, Response, Result, RoutedLlmClient, Score,
};
use switchyard_protocol::{completion_text, text_request, text_response};
@@ -45,7 +45,7 @@ struct AlwaysOrchestrator;
impl Classifier for AlwaysOrchestrator {
async fn score(
&self,
- _state: &mut State,
+ _state: &mut (),
_request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
@@ -85,24 +85,19 @@ fn request(headers: &[(&str, &str)]) -> Request {
/// terminal classifier serves everything else.
fn router() -> Arc<FallThrough> {
Arc::new(
- FallThrough::new(targets())
+ FallThrough::<()>::new(targets())
.with_component(Arc::new(AffinityRouter::for_subagents()))
.with_classifier(Arc::new(SubagentOverride::new("worker")))
.with_classifier(Arc::new(AlwaysOrchestrator)),
)
}
-/// Runs one turn on `ctx`, returning the target that served it.
-///
-/// `ctx` carries the per-session [`State`] the processor chain folds into. The affinity
-/// assignments are held on the `AffinityRouter` instance instead, so replay follows the
-/// shared router rather than the context.
-async fn turn(
- router: &Arc<FallThrough>,
- ctx: Context<SharedState>,
- headers: &[(&str, &str)],
-) -> Result<String> {
- let (_, response) = router.clone().run(ctx, request(headers)).await?;
+/// Runs one turn, returning the target that served it.
+async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {
+ let (_, response) = router
+ .clone()
+ .run(Context::default(), request(headers))
+ .await?;
Ok(response
.llm_response
.as_agg()
@@ -120,53 +115,42 @@ fn child(agent: &str) -> Vec<(&str, &str)> {
#[tokio::test]
async fn root_traffic_falls_through_to_the_terminal_classifier() -> Result<()> {
- let ctx = Context::<SharedState>::default();
// No sub-agent lineage: affinity abstains (subagents only), the override abstains, and
// the terminal classifier serves the turn.
- let served = turn(&router(), ctx, &[("x-claude-code-session-id", "session-1")]).await?;
+ let served = turn(&router(), &[("x-claude-code-session-id", "session-1")]).await?;
assert_eq!(served, "orchestrator");
Ok(())
}
#[tokio::test]
async fn delegated_work_is_routed_to_the_worker() -> Result<()> {
- let ctx = Context::<SharedState>::default();
- assert_eq!(turn(&router(), ctx, &child("child-1")).await?, "worker");
+ assert_eq!(turn(&router(), &child("child-1")).await?, "worker");
Ok(())
}
#[tokio::test]
async fn the_override_seeds_a_pin_that_affinity_replays() -> Result<()> {
let router = router();
- let ctx = Context::<SharedState>::default();
// Turn 1: affinity has no assignment, so the override decides and the decision is
// replayed into affinity.
- assert_eq!(
- turn(&router, ctx.clone(), &child("child-1")).await?,
- "worker"
- );
+ assert_eq!(turn(&router, &child("child-1")).await?, "worker");
// Turns 2-3: the lineage header still identifies the child, but affinity — not the
// override — is now the classifier that answers, because it scores first.
- assert_eq!(
- turn(&router, ctx.clone(), &child("child-1")).await?,
- "worker"
- );
- assert_eq!(turn(&router, ctx, &child("child-1")).await?, "worker");
+ assert_eq!(turn(&router, &child("child-1")).await?, "worker");
+ assert_eq!(turn(&router, &child("child-1")).await?, "worker");
Ok(())
}
#[tokio::test]
async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()> {
let router = router();
- let ctx = Context::<SharedState>::default();
// A Codex `compact` turn is sub-agent *lineage* but not delegated *work*: the two
// predicates differ on purpose, so the override abstains and it routes normally.
let served = turn(
&router,
- ctx,
&[
("x-codex-session-id", "session-1"),
("x-openai-subagent", "compact"),
@@ -180,7 +164,7 @@ async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()>
/// Builds a cascade whose override scores `worker`, sharing `affinity` across instances.
fn router_overriding_to(affinity: Arc<AffinityRouter>, worker: &str) -> Arc<FallThrough> {
Arc::new(
- FallThrough::new(targets())
+ FallThrough::<()>::new(targets())
.with_component(affinity)
.with_classifier(Arc::new(SubagentOverride::new(worker)))
.with_classifier(Arc::new(AlwaysOrchestrator)),
@@ -197,14 +181,8 @@ async fn the_pin_outlives_the_policy_that_seeded_it() -> Result<()> {
let seed = router_overriding_to(affinity.clone(), "worker");
let rebound = router_overriding_to(affinity, "reviewer");
- assert_eq!(
- turn(&seed, Context::default(), &child("child-1")).await?,
- "worker"
- );
- assert_eq!(
- turn(&rebound, Context::default(), &child("child-1")).await?,
- "worker"
- );
+ assert_eq!(turn(&seed, &child("child-1")).await?, "worker");
+ assert_eq!(turn(&rebound, &child("child-1")).await?, "worker");
Ok(())
}
@@ -216,14 +194,8 @@ async fn without_a_shared_pin_the_second_policy_wins() -> Result<()> {
let seed = router_overriding_to(Arc::new(AffinityRouter::for_subagents()), "worker");
let rebound = router_overriding_to(Arc::new(AffinityRouter::for_subagents()), "reviewer");
- assert_eq!(
- turn(&seed, Context::default(), &child("child-1")).await?,
- "worker"
- );
- assert_eq!(
- turn(&rebound, Context::default(), &child("child-1")).await?,
- "reviewer"
- );
+ assert_eq!(turn(&seed, &child("child-1")).await?, "worker");
+ assert_eq!(turn(&rebound, &child("child-1")).await?, "reviewer");
Ok(())
}
@@ -237,20 +209,11 @@ async fn distinct_children_are_pinned_independently() -> Result<()> {
let seed = router_overriding_to(affinity.clone(), "worker");
let sibling = router_overriding_to(affinity, "reviewer");
- assert_eq!(
- turn(&seed, Context::default(), &child("child-1")).await?,
- "worker"
- );
- assert_eq!(
- turn(&sibling, Context::default(), &child("child-2")).await?,
- "reviewer"
- );
+ assert_eq!(turn(&seed, &child("child-1")).await?, "worker");
+ assert_eq!(turn(&sibling, &child("child-2")).await?, "reviewer");
// child-1's own pin is untouched by the sibling's assignment: affinity replays it
// even through the cascade whose override would otherwise score "reviewer".
- assert_eq!(
- turn(&sibling, Context::default(), &child("child-1")).await?,
- "worker"
- );
+ assert_eq!(turn(&sibling, &child("child-1")).await?, "worker");
Ok(())
}
@@ -259,11 +222,10 @@ async fn a_cascade_without_the_override_still_routes_root_traffic() -> Result<()
// Affinity and the override are independent: dropping the override leaves a valid
// cascade, which is the point of composing them rather than nesting one in the other.
let router = Arc::new(
- FallThrough::new(targets())
+ FallThrough::<()>::new(targets())
.with_component(Arc::new(AffinityRouter::for_subagents()))
.with_classifier(Arc::new(AlwaysOrchestrator)),
);
- let ctx = Context::<SharedState>::default();
- assert_eq!(turn(&router, ctx, &child("child-1")).await?, "orchestrator");
+ assert_eq!(turn(&router, &child("child-1")).await?, "orchestrator");
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index 5d809645c..8f33db35a 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -7,9 +7,8 @@
//! forces that model on later requests sharing the identity. It is one object that plays
//! both SDK roles, so registering it as a processor and a classifier cannot drift apart:
//!
-//! - As a [`Processor`] it *writes* the assignment: it captures the request's identity on
-//! [`Event::Request`] and binds it to the chosen model on [`Event::Decision`], the first
-//! assignment for an identity winning.
+//! - As a [`Processor`] it *writes* the assignment: [`Event::Decision`] carries the request
+//! whose chosen model is retained, with the first assignment for an identity winning.
//! - As a [`Classifier`] it *reads* the assignment: it scores the retained model with
//! confidence `1.0`, or returns no scores to abstain.
//!
@@ -24,7 +23,7 @@ use async_trait::async_trait;
use parking_lot::Mutex;
use switchyard_protocol::Request;
-use crate::{Classification, Classifier, Event, Processor, Score, State};
+use crate::{Classification, Classifier, Event, Processor, Score};
/// Upper bound on retained assignments, keeping the process-local map from growing
/// without limit; the oldest entry is evicted once the bound is reached.
@@ -42,23 +41,12 @@ enum AffinityKey {
Subagent { session: String, agent: String },
}
-/// The affinity memory: the retained model per identity, plus the key captured from the
-/// current turn's request while its decision is pending.
-#[derive(Default)]
-struct AffinityAssignments {
- /// Identity → retained model, first assignment winning.
- assignments: HashMap<AffinityKey, String>,
- /// Key captured on this turn's [`Event::Request`], consumed by its [`Event::Decision`].
- pending: Option<AffinityKey>,
-}
-
/// Retains a model per request identity and forces it on later matching requests.
///
/// Register the same instance as both a processor and a classifier; the two roles share
-/// the retained assignments through this instance's interior storage. Because a decision
-/// event carries only the chosen model, the request's identity is captured first (on
-/// [`Event::Request`]) and consumed when the decision arrives — this assumes a turn's
-/// request is observed before its decision, as the driving algorithm folds a turn in order.
+/// the retained assignments through this instance's interior storage. Decision events carry
+/// their originating request, so concurrent turns cannot bind one request's identity to
+/// another request's selected model.
///
/// [`with_latch_only`](Self::with_latch_only) narrows *which* models are retained — a
/// decision for any other model routes normally but is not latched (the escalation latch:
@@ -71,10 +59,9 @@ pub struct AffinityRouter {
subagents_only: bool,
/// Retained assignments, shared across this router's processor and classifier roles.
///
- /// Held on the instance rather than in [`State`] so the two roles share one
- /// process-local map through a single registered [`Arc`](std::sync::Arc); bounded by
- /// [`MAX_ASSIGNMENTS`].
- assignments: Mutex<AffinityAssignments>,
+ /// Held on the instance so the two roles share one process-local map through a
+ /// single registered [`Arc`](std::sync::Arc); bounded by [`MAX_ASSIGNMENTS`].
+ assignments: Mutex<HashMap<AffinityKey, String>>,
}
impl AffinityRouter {
@@ -125,45 +112,40 @@ impl AffinityRouter {
}
#[async_trait]
-impl Processor for AffinityRouter {
- async fn process(&self, _state: &mut State, event: Event<'_>) -> crate::Result<()> {
- // No `.await` is held across this guard.
- let mut fact = self.assignments.lock();
- match event {
- // Capture the identity now; the model it maps to is only known at the decision.
- Event::Request(request) => {
- fact.pending = self.affinity_key(request);
- }
- // Bind the captured identity to the chosen model, keeping any earlier winner.
- Event::Decision(decision) => {
+impl<S> Processor<S> for AffinityRouter
+where
+ S: Send + 'static,
+{
+ async fn process(&self, _state: &mut S, event: Event<'_>) -> crate::Result<()> {
+ if let Event::Decision { request, decision } = event {
+ if let Some(key) = self.affinity_key(request) {
let model = decision.selected_model();
- let Some(key) = fact.pending.take() else {
- return Ok(());
- };
- // Latch only permitted models; others consume `pending` but are not retained.
- if self.should_latch(model) && !fact.assignments.contains_key(&key) {
- evict_if_full(&mut fact.assignments);
- fact.assignments.insert(key, model.to_string());
+ let mut assignments = self.assignments.lock();
+ if self.should_latch(model) && !assignments.contains_key(&key) {
+ evict_if_full(&mut assignments);
+ assignments.insert(key, model.to_string());
}
}
- _ => {}
}
Ok(())
}
}
#[async_trait]
-impl Classifier for AffinityRouter {
+impl<S> Classifier<S> for AffinityRouter
+where
+ S: Send + 'static,
+{
async fn score(
&self,
- _state: &mut State,
+ _state: &mut S,
request: &mut Request,
_driver: Option<&crate::Driver>,
) -> crate::Result<Classification> {
let Some(key) = self.affinity_key(request) else {
return Ok(Classification::Scores(Vec::new()));
};
- let assigned = self.assignments.lock().assignments.get(&key).cloned();
+ let assigned = self.assignments.lock().get(&key).cloned();
Ok(Classification::Scores(match assigned {
Some(target) => vec![Score {
confidence: 1.0,
@@ -189,10 +171,9 @@ mod tests {
use std::sync::Arc;
- use switchyard_protocol::{text_request, Decision, Metadata, Signals};
+ use switchyard_protocol::{text_request, Decision, Metadata};
- /// Boxed, thread-safe error type keeping the test helpers ergonomic; libsy errors
- /// convert into it through `?`.
+ /// Boxed, thread-safe error type keeping the test helpers ergonomic.
type BoxErr = Box<dyn std::error::Error + Send + Sync>;
/// A decision that reports a fixed selected model.
@@ -241,13 +222,18 @@ mod tests {
/// Folds a request and its decision through the router, retaining `model`.
async fn retain(
router: &AffinityRouter,
- state: &mut State,
+ state: &mut (),
request: &mut Request,
model: &'static str,
) -> Result<(), BoxErr> {
- router.process(state, Event::Request(request)).await?;
router
- .process(state, Event::Decision(&FixedDecision(model)))
+ .process(
+ state,
+ Event::Decision {
+ request,
+ decision: &FixedDecision(model),
+ },
+ )
.await?;
Ok(())
}
@@ -255,7 +241,7 @@ mod tests {
/// Scores through the definitive classification variant used by affinity.
async fn scores(
classifier: &dyn Classifier,
- state: &mut State,
+ state: &mut (),
request: &mut Request,
) -> Result<Vec<Score>, BoxErr> {
match classifier.score(state, request, None).await? {
@@ -267,7 +253,7 @@ mod tests {
#[tokio::test]
async fn session_retains_first_model_across_requests() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
let mut first = request(session("session-1", "agent-a"));
retain(&router, &mut state, &mut first, "model-a").await?;
@@ -284,7 +270,7 @@ mod tests {
#[tokio::test]
async fn subagent_only_retains_children_without_latching_root_traffic() -> Result<(), BoxErr> {
let router = AffinityRouter::for_subagents();
- let mut state = State::default();
+ let mut state = ();
let mut root = request(session("session-1", "root-agent"));
retain(&router, &mut state, &mut root, "model-a").await?;
@@ -304,7 +290,7 @@ mod tests {
#[tokio::test]
async fn first_decision_wins() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
let mut req = request(session("session-1", "agent-a"));
retain(&router, &mut state, &mut req, "model-a").await?;
@@ -322,7 +308,7 @@ mod tests {
#[tokio::test]
async fn subagent_is_keyed_by_agent_not_task() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
let mut first = request(subagent("child-1", "task-1"));
retain(&router, &mut state, &mut first, "model-a").await?;
@@ -340,7 +326,7 @@ mod tests {
#[tokio::test]
async fn distinct_subagents_are_assigned_independently() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
// One child in the session is pinned...
retain(
@@ -360,7 +346,7 @@ mod tests {
#[tokio::test]
async fn subagent_does_not_inherit_session_assignment() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
// The session root is pinned, but a sub-agent is keyed separately...
retain(
@@ -380,7 +366,7 @@ mod tests {
#[tokio::test]
async fn classifier_abstains_without_a_session() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
// No session id at all: nothing to key on.
let mut req = request(Metadata::default());
@@ -395,14 +381,17 @@ mod tests {
let router = Arc::new(AffinityRouter::new());
let processor: Arc<dyn Processor> = router.clone();
let classifier: Arc<dyn Classifier> = router;
- let mut state = State::default();
+ let mut state = ();
- let mut first = request(session("session-1", "agent-a"));
+ let first = request(session("session-1", "agent-a"));
processor
- .process(&mut state, Event::Request(&mut first))
- .await?;
- processor
- .process(&mut state, Event::Decision(&FixedDecision("model-a")))
+ .process(
+ &mut state,
+ Event::Decision {
+ request: &first,
+ decision: &FixedDecision("model-a"),
+ },
+ )
.await?;
let mut second = request(session("session-1", "agent-b"));
@@ -415,13 +404,19 @@ mod tests {
}
#[tokio::test]
- async fn decision_without_a_request_is_ignored() -> Result<(), BoxErr> {
+ async fn decision_without_an_affinity_identity_is_ignored() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
+ let unkeyed = request(Metadata::default());
- // A decision with no captured identity (no prior request) stores nothing.
router
- .process(&mut state, Event::Decision(&FixedDecision("model-a")))
+ .process(
+ &mut state,
+ Event::Decision {
+ request: &unkeyed,
+ decision: &FixedDecision("model-a"),
+ },
+ )
.await?;
let mut req = request(session("session-1", "agent-a"));
@@ -430,33 +425,50 @@ mod tests {
}
#[tokio::test]
- async fn unrelated_events_preserve_the_pending_identity() -> Result<(), BoxErr> {
+ async fn decisions_retain_their_originating_request_identity() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
+ let mut first = request(session("session-1", "agent-a"));
+ let mut second = request(session("session-2", "agent-b"));
- let mut req = request(session("session-1", "agent-a"));
- router.process(&mut state, Event::Request(&mut req)).await?;
- // A stray signal between the request and its decision must not drop the captured
- // identity, nor create an assignment of its own.
+ // Replay decisions in the opposite order. Each decision carries its request,
+ // so the assignments cannot cross.
router
- .process(&mut state, Event::Signal(&Signals {}))
+ .process(
+ &mut state,
+ Event::Decision {
+ request: &second,
+ decision: &FixedDecision("model-b"),
+ },
+ )
.await?;
router
- .process(&mut state, Event::Decision(&FixedDecision("model-a")))
+ .process(
+ &mut state,
+ Event::Decision {
+ request: &first,
+ decision: &FixedDecision("model-a"),
+ },
+ )
.await?;
- let scores = scores(&router, &mut state, &mut req).await?;
+ let first_scores = scores(&router, &mut state, &mut first).await?;
+ let second_scores = scores(&router, &mut state, &mut second).await?;
assert_eq!(
- scores.first().map(|score| score.target.as_str()),
+ first_scores.first().map(|score| score.target.as_str()),
Some("model-a")
);
+ assert_eq!(
+ second_scores.first().map(|score| score.target.as_str()),
+ Some("model-b")
+ );
Ok(())
}
#[tokio::test]
async fn distinct_sessions_are_assigned_independently() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
retain(
&router,
@@ -499,7 +511,7 @@ mod tests {
#[tokio::test]
async fn subagent_without_an_agent_id_is_not_keyed() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
// The sub-agent flag is set but no agent id is present, so no key can be formed;
// the request is neither retained nor scored.
@@ -517,7 +529,7 @@ mod tests {
#[tokio::test]
async fn assignments_are_bounded_by_the_cap() -> Result<(), BoxErr> {
let router = AffinityRouter::new();
- let mut state = State::default();
+ let mut state = ();
// One distinct session past the cap forces exactly one eviction.
for index in 0..=MAX_ASSIGNMENTS {
@@ -531,7 +543,7 @@ mod tests {
.await?;
}
- let len = router.assignments.lock().assignments.len();
+ let len = router.assignments.lock().len();
assert_eq!(len, MAX_ASSIGNMENTS);
Ok(())
}
@@ -539,7 +551,7 @@ mod tests {
#[tokio::test]
async fn latch_only_retains_matching_models() -> Result<(), BoxErr> {
let router = AffinityRouter::new().with_latch_only(["strong"]);
- let mut state = State::default();
+ let mut state = ();
let mut req = request(session("session-1", "agent-a"));
// A "weak" decision is not retained — a later turn is not latched.
diff --git a/crates/libsy/src/algorithms/util/handoff_notes.rs b/crates/libsy/src/algorithms/util/handoff_notes.rs
index dbaba4c67..bbf9ede02 100644
--- a/crates/libsy/src/algorithms/util/handoff_notes.rs
+++ b/crates/libsy/src/algorithms/util/handoff_notes.rs
@@ -67,9 +67,9 @@ impl HandoffNoteProcessor {
}
#[async_trait]
-impl Processor for HandoffNoteProcessor {
+impl Processor<State> for HandoffNoteProcessor {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
- let Event::Decision(decision) = event else {
+ let Event::Decision { decision, .. } = event else {
return Ok(());
};
let tier = decision.selected_model().to_string();
@@ -90,7 +90,7 @@ impl Processor for HandoffNoteProcessor {
#[cfg(test)]
mod tests {
use super::*;
- use crate::Decision;
+ use crate::{Decision, Request};
struct FakeDecision(&'static str);
impl Decision for FakeDecision {
@@ -115,8 +115,16 @@ mod tests {
}
async fn run(processor: &HandoffNoteProcessor, state: &mut State, tier: &'static str) {
+ let request = Request::default();
+ let decision = FakeDecision(tier);
processor
- .process(state, Event::Decision(&FakeDecision(tier)))
+ .process(
+ state,
+ Event::Decision {
+ request: &request,
+ decision: &decision,
+ },
+ )
.await
.unwrap();
}
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 04b29cdd8..e876debe4 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -118,7 +118,7 @@ where
}
#[async_trait]
-impl<J, P> Classifier for JudgeClassifier<J, P>
+impl<J, P> Classifier<State> for JudgeClassifier<J, P>
where
J: Judge,
P: JudgePolicy<Verdict = J::Verdict>,
@@ -381,8 +381,9 @@ mod tests {
#[tokio::test]
async fn a_missing_driver_is_an_error_not_a_fallback() -> Result<()> {
+ let mut request = request();
let error = classifier()
- .score(&mut State::default(), &mut request(), None)
+ .score(&mut State::default(), &mut request, None)
.await
.err()
.ok_or_else(|| LibsyError::AlgorithmError {
diff --git a/crates/libsy/src/algorithms/util/stage_router.rs b/crates/libsy/src/algorithms/util/stage_router.rs
index c8edd4062..a9d69251c 100644
--- a/crates/libsy/src/algorithms/util/stage_router.rs
+++ b/crates/libsy/src/algorithms/util/stage_router.rs
@@ -305,7 +305,7 @@ impl StageClassifier {
}
#[async_trait]
-impl Classifier for StageClassifier {
+impl Classifier<State> for StageClassifier {
async fn score(
&self,
state: &mut State,
diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs
index aa3b4e598..166c8f121 100644
--- a/crates/libsy/src/algorithms/util/subagent.rs
+++ b/crates/libsy/src/algorithms/util/subagent.rs
@@ -7,8 +7,8 @@
//! sub-agent work ([`Metadata::is_subagent_work`]) and abstains for everything else, so a
//! cascade falls through to its later classifiers on ordinary traffic.
//!
-//! It is stateless and holds only the worker's *name*: the
-//! [`FallThrough`](crate::algorithms::FallThrough) cascade resolves that name against its
+//! It is stateless and holds only the worker's *name*: the fall-through cascade resolves it
+//! against its
//! target set. Keeping the policy independent of any memory of past decisions is what lets
//! it compose with a stateful classifier such as
//! [`AffinityRouter`](crate::algorithms::AffinityRouter) — the override decides *which*
@@ -17,7 +17,7 @@
use async_trait::async_trait;
-use crate::{Classification, Classifier, Driver, Metadata, Request, Result, Score, State};
+use crate::{Classification, Classifier, Driver, Metadata, Request, Result, Score};
/// Scores a fixed worker target for delegated sub-agent work; abstains otherwise.
pub struct SubagentOverride {
@@ -38,10 +38,13 @@ impl SubagentOverride {
}
#[async_trait]
-impl Classifier for SubagentOverride {
+impl<S> Classifier<S> for SubagentOverride
+where
+ S: Send + 'static,
+{
async fn score(
&self,
- _state: &mut State,
+ _state: &mut S,
request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
@@ -87,7 +90,7 @@ mod tests {
/// Scores `headers` through the override, returning the winning target if it scored.
async fn selected(headers: &[(&str, &str)]) -> Result<Option<String>> {
- let mut state = State::default();
+ let mut state = ();
let classification = SubagentOverride::new("worker")
.score(&mut state, &mut request(headers), None)
.await?;
@@ -135,7 +138,7 @@ mod tests {
async fn delegated_work_is_scored_definitively() -> Result<()> {
// Confidence 1.0 under `Scores` (never `Ambiguous`), so the cascade stops here
// rather than consulting later classifiers.
- let mut state = State::default();
+ let mut state = ();
let classification = SubagentOverride::new("worker")
.score(
&mut state,
diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rs
index 143d1044c..1b0a0d9f2 100644
--- a/crates/libsy/src/algorithms/util/tool_signals.rs
+++ b/crates/libsy/src/algorithms/util/tool_signals.rs
@@ -274,7 +274,7 @@ impl Default for ToolSignalProcessor {
}
#[async_trait]
-impl Processor for ToolSignalProcessor {
+impl Processor<State> for ToolSignalProcessor {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Request(req) = event {
let tool_signal = ToolSignals::from_request(req, Some(self.recent_window));
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index b3e71bc4e..0d329601a 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -327,19 +327,9 @@ impl LlmTargetSet {
/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).
///
/// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across
-/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms
-/// (the reference routers) get this for free; a stateful one keeps its per-session state
-/// in the threaded [`Context<S>`], not in the shared algorithm.
-///
-/// The `S` type parameter is the per-session state carried in `Context<S>`. It defaults
-/// to `()`, so stateless algorithms just write `impl Algorithm for MyRouter` and callers
-/// keep using `Arc<dyn Algorithm>`. A stateful algorithm picks its own state type (e.g.
-/// `impl Algorithm<SharedState> for FallThrough`).
+/// requests and run concurrently, so it owns its thread-safety and any shared state.
#[async_trait]
-pub trait Algorithm<S = ()>: Send + Sync + 'static
-where
- S: Clone + Send + Sync + 'static,
-{
+pub trait Algorithm: Send + Sync + 'static {
/// Stable, low-cardinality name identifying this algorithm — the
/// `algorithm` attribute on every span, metric, and log line the crate
/// emits for its runs (see the crate docs' Observability section).
@@ -349,10 +339,10 @@ where
/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`].
/// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)
/// drive it. `ctx` carries the request's cross-cutting values (today: the
- /// algorithm's telemetry label in [`Context::values`]) and per-session `state`.
+ /// algorithm's telemetry label in [`Context::values`]).
async fn create_run_task(
self: Arc<Self>,
- ctx: Context<S>,
+ ctx: Context,
driver: Driver,
request: Request,
) -> Result<Response>;
@@ -401,7 +391,7 @@ where
/// Process a request to completion, returning a stream of [`Step`]s.
/// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve.
/// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.
- fn run_stream(self: Arc<Self>, ctx: Context<S>, request: Request) -> StepStream {
+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {
// Stamp the algorithm's telemetry label into the request context; the
// context rides on every driver call, so its telemetry is attributed.
let mut ctx = ctx;
@@ -431,8 +421,7 @@ where
let abort_guard = AbortOnDrop(handle.abort_handle());
let finish_driver = driver.clone();
- // The terminal step carries no session state; hand `finish` the base context.
- let finish_ctx = ctx.without_state();
+ let finish_ctx = ctx;
let tail: StepStream = Box::pin(
futures::stream::once(async move {
let result = match handle.await {
@@ -456,7 +445,7 @@ where
/// [`Decision`]s the algorithm made along the way.
async fn run(
self: Arc<Self>,
- ctx: Context<S>,
+ ctx: Context,
request: Request,
) -> Result<(Vec<Arc<dyn Decision>>, Response)> {
// Serve one offloaded call with its target's default client. A failed *model*
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index 0d949ddb1..2ab90f6a0 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-use crate::{Driver, LibsyError, Result, State};
+use crate::{Driver, LibsyError, Result};
use async_trait::async_trait;
use switchyard_protocol::Request;
@@ -67,9 +67,9 @@ fn argmax(scores: &[Score]) -> Result<Option<Score>> {
Ok(best.cloned())
}
-/// Scores each of the classifier's targets given State the current Request
+/// Scores targets from the current request and the composition's state.
#[async_trait]
-pub trait Classifier: Send + Sync {
+pub trait Classifier<S = ()>: Send + Sync {
/// Stable tier represented by `selected_model`, when this classifier defines one.
fn routing_tier(&self, _selected_model: &str) -> Option<&'static str> {
None
@@ -85,7 +85,7 @@ pub trait Classifier: Send + Sync {
/// only read it.
async fn score(
&self,
- state: &mut State,
+ state: &mut S,
request: &mut Request,
driver: Option<&Driver>,
) -> Result<Classification>;
@@ -94,7 +94,6 @@ pub trait Classifier: Send + Sync {
#[cfg(test)]
mod tests {
use super::*;
- use crate::StateValue;
use switchyard_protocol::text_request;
/// Terse `Score` builder for the assertions below.
@@ -178,15 +177,14 @@ mod tests {
struct RecordingClassifier;
#[async_trait]
- impl Classifier for RecordingClassifier {
+ impl Classifier<bool> for RecordingClassifier {
async fn score(
&self,
- state: &mut State,
+ state: &mut bool,
request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
- // Stash a marker in `extra` to prove state is threaded mutably.
- state.extra.insert("ran".to_string(), StateValue::Count(1));
+ *state = true;
let target = request.requested_model().unwrap_or("auto").to_string();
Ok(Classification::Scores(vec![Score {
target,
@@ -197,7 +195,7 @@ mod tests {
#[tokio::test]
async fn classifier_reads_request_and_mutates_state() -> Result<()> {
- let mut state = State::default();
+ let mut state = false;
let mut request = Request {
llm_request: text_request(Some("strong".to_string()), "hi"),
raw_request: None,
@@ -211,7 +209,7 @@ mod tests {
classification.argmax(false)?.map(|s| s.target),
Some("strong".to_string())
);
- assert!(matches!(state.extra.get("ran"), Some(StateValue::Count(1))));
+ assert!(state);
Ok(())
}
@@ -222,7 +220,7 @@ mod tests {
impl Classifier for RewritingClassifier {
async fn score(
&self,
- _state: &mut State,
+ _state: &mut (),
request: &mut Request,
_driver: Option<&Driver>,
) -> Result<Classification> {
@@ -236,7 +234,7 @@ mod tests {
#[tokio::test]
async fn classifier_rewrites_the_request_in_place() -> Result<()> {
- let mut state = State::default();
+ let mut state = ();
let mut request = Request {
llm_request: text_request(Some("auto".to_string()), "hi"),
raw_request: None,
diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs
index b21cc4c23..d3bb7856c 100644
--- a/crates/libsy/src/core/processor.rs
+++ b/crates/libsy/src/core/processor.rs
@@ -1,11 +1,11 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-use crate::{Result, State};
+use crate::Result;
use async_trait::async_trait;
use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals};
-/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate [`State`]
+/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate state.
///
/// The two request-bearing variants borrow the request mutably, so a processor may rewrite
/// it in place and pass the rewritten request down the chain (see [`Processor::process`]).
@@ -15,63 +15,61 @@ pub enum Event<'a> {
Request(&'a mut Request),
/// An out-of-band agentic-stack signal (tool results, budget updates, …).
Signal(&'a Signals),
- /// A routing decision the algorithm just made.
- Decision(&'a dyn Decision),
+ /// A routing decision paired with the request that produced it.
+ Decision {
+ /// The request classified by the algorithm.
+ request: &'a Request,
+ /// The routing decision produced for `request`.
+ decision: &'a dyn Decision,
+ },
/// A request about to be sent to a model.
ModelRequest(&'a mut Request),
/// A buffered response received back from a model.
ModelResponse(&'a AggLlmResponse),
}
-/// Collects events as the algorithm runs and mutates [`State`]
+/// Collects events as the algorithm runs and mutates the composition's state.
#[async_trait]
-pub trait Processor: Send + Sync {
+pub trait Processor<S = ()>: Send + Sync {
/// Process an event, accumulating facts into `state`.
///
/// A request-bearing event ([`Event::Request`], [`Event::ModelRequest`]) may also be
/// rewritten in place; the edit propagates to the rest of the chain and to the model
/// call. Most processors only read it.
- async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()>;
+ async fn process(&self, state: &mut S, event: Event<'_>) -> Result<()>;
}
#[cfg(test)]
mod tests {
use super::*;
- use crate::StateValue;
+ use std::collections::HashMap;
use switchyard_protocol::{text_request, text_response};
- /// The `State::extra` key each event variant tallies under.
+ type TestState = HashMap<&'static str, u32>;
+
+ /// The key each event variant tallies under.
fn event_key(event: &Event<'_>) -> &'static str {
match event {
Event::Request(_) => "requests",
Event::Signal(_) => "signals",
- Event::Decision(_) => "decisions",
+ Event::Decision { .. } => "decisions",
Event::ModelRequest(_) => "model_requests",
Event::ModelResponse(_) => "model_responses",
}
}
- /// Reads a `StateValue::Count` from `extra`, treating a missing key as zero.
- fn count(state: &State, key: &str) -> u32 {
- match state.extra.get(key) {
- Some(StateValue::Count(n)) => *n,
- _ => 0,
- }
+ /// Reads a count, treating a missing key as zero.
+ fn count(state: &TestState, key: &'static str) -> u32 {
+ state.get(key).copied().unwrap_or_default()
}
- /// Tallies each event variant under its own key in [`State::extra`].
+ /// Tallies each event variant under its own key.
struct CountingProcessor;
#[async_trait]
- impl Processor for CountingProcessor {
- async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
- let entry = state
- .extra
- .entry(event_key(&event).to_string())
- .or_insert(StateValue::Count(0));
- if let StateValue::Count(n) = entry {
- *n += 1;
- }
+ impl Processor<TestState> for CountingProcessor {
+ async fn process(&self, state: &mut TestState, event: Event<'_>) -> Result<()> {
+ *state.entry(event_key(&event)).or_default() += 1;
Ok(())
}
}
@@ -102,7 +100,7 @@ mod tests {
#[tokio::test]
async fn processor_tallies_each_event_variant_into_state() -> Result<()> {
let processor = CountingProcessor;
- let mut state = State::default();
+ let mut state = TestState::default();
let mut req = request();
let response = text_response(None, "ok");
let decision = TestDecision;
@@ -119,7 +117,13 @@ mod tests {
.process(&mut state, Event::ModelResponse(&response))
.await?;
processor
- .process(&mut state, Event::Decision(&decision))
+ .process(
+ &mut state,
+ Event::Decision {
+ request: &req,
+ decision: &decision,
+ },
+ )
.await?;
processor
.process(&mut state, Event::Signal(&signals))
@@ -136,7 +140,7 @@ mod tests {
#[tokio::test]
async fn process_accumulates_state_across_repeated_events() -> Result<()> {
let processor = CountingProcessor;
- let mut state = State::default();
+ let mut state = TestState::default();
let mut req = request();
for _ in 0..3 {
@@ -154,7 +158,7 @@ mod tests {
#[async_trait]
impl Processor for RewritingProcessor {
- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {
+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {
if let Event::Request(request) | Event::ModelRequest(request) = event {
request.llm_request.model = Some("rewritten".to_string());
}
@@ -164,7 +168,7 @@ mod tests {
#[tokio::test]
async fn processor_rewrites_the_request_in_place() -> Result<()> {
- let mut state = State::default();
+ let mut state = ();
let mut req = request();
assert_eq!(req.requested_model(), Some("auto"));
diff --git a/crates/libsy/src/core/state.rs b/crates/libsy/src/core/state.rs
index 58382c76d..b6be3e4ca 100644
--- a/crates/libsy/src/core/state.rs
+++ b/crates/libsy/src/core/state.rs
@@ -1,10 +1,9 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-use std::collections::HashMap;
-use std::sync::Arc;
+//! Built-in state shared by processors and classifiers across a session.
-use tokio::sync::Mutex;
+use std::collections::HashMap;
use crate::ToolSignals;
@@ -17,7 +16,7 @@ pub enum StateValue {
Scalar(f32),
}
-/// State maintained by [`Algorithm`](crate::Algorithm)s
+/// Routing facts accumulated across one session's algorithm runs.
#[derive(Debug, Clone, Default)]
pub struct State {
pub turn_count: u32,
@@ -27,6 +26,3 @@ pub struct State {
pub tool_signals: Option<ToolSignals>,
pub extra: HashMap<String, StateValue>,
}
-
-/// State shared accross tunrs / sessions
-pub type SharedState = Arc<Mutex<State>>;
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index a2eeae382..09f61a793 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -53,7 +53,7 @@ static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = On
pub(crate) const ALGORITHM_KEY: &str = "algorithm";
/// The algorithm label carried by a request context; empty until stamped.
-pub(crate) fn algorithm_label<S>(ctx: &Context<S>) -> &str {
+pub(crate) fn algorithm_label(ctx: &Context) -> &str {
ctx.values
.get(ALGORITHM_KEY)
.map(String::as_str)
@@ -135,8 +135,8 @@ pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {
/// Runs one algorithm task to completion, recording the run counter, duration
/// histogram, span outcome, and failure log when it resolves. Executes inside
/// the `libsy.run` span its caller instruments the task with.
-pub(crate) async fn observe_run<S>(
- ctx: Context<S>,
+pub(crate) async fn observe_run(
+ ctx: Context,
run: impl Future<Output = Result<Response>>,
) -> Result<Response> {
let started = Instant::now();
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index 63f2cd6a5..70dee0411 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -29,7 +29,7 @@ use tracing_subscriber::Layer;
use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};
use switchyard_libsy::{
AggLlmResponse, Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmTarget,
- LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, SharedState, Step, Usage,
+ LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, State, Step, Usage,
};
use switchyard_protocol::{text_request, text_response, LlmClientError};
@@ -725,13 +725,15 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib
let targets = LlmTargetSet::new(vec![target("weak"), target("strong")]);
let weak = targets.get_target("weak")?;
let strong = targets.get_target("strong")?;
- let router = Arc::new(FallThrough::new(targets).with_classifier(Arc::new(
- LlmTaskClassifier::new(target("classifier"), weak, strong, 0.5)?,
- )));
+ let router = Arc::new(
+ FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(
+ LlmTaskClassifier::new(target("classifier"), weak, strong, 0.5)?,
+ )),
+ );
let (trace, _response) = router
.run(
- Context::<SharedState>::default(),
+ Context::default(),
Request {
llm_request: text_request(Some("auto".to_string()), "classify this"),
raw_request: None,
diff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rs
index 4560ef7f3..570f3873f 100644
--- a/crates/protocol/src/envelope.rs
+++ b/crates/protocol/src/envelope.rs
@@ -7,24 +7,11 @@
use crate::{LlmRequest, LlmResponse, Metadata};
use std::collections::HashMap;
-/// [`Context`] for an algorithm or llm client.
-/// This struct is fed through the entire process and provides unified information to each step
+/// Cross-cutting request context passed through algorithms and LLM clients.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
-pub struct Context<S = ()> {
- /// caller specific values
+pub struct Context {
+ /// Caller-specific values propagated through the request.
pub values: HashMap<String, String>,
- /// Caller-defined per-session state; `()` when a consumer carries no session state.
- pub state: S,
-}
-
-impl<S> Context<S> {
- /// Create a new [`Context`] with the given state and an empty values map.
- pub fn without_state(&self) -> Context<()> {
- Context {
- values: self.values.clone(),
- state: (),
- }
- }
}
/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budget
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index 0c6ac9559..3315a8914 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -9,7 +9,7 @@ use std::path::Path;
use std::sync::Arc;
use libsy::algorithms::{FallThrough, LlmTaskClassifier, Noop, Passthrough, Random};
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State};
use serde::Deserialize;
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
@@ -240,7 +240,7 @@ fn build_algorithm(
route_name: &str,
config: &RouteConfig,
targets: &BTreeMap<String, LlmTarget>,
-) -> ServerResult<Arc<dyn Algorithm<SharedState>>> {
+) -> ServerResult<Arc<dyn Algorithm>> {
match config {
RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),
RouteConfig::Random {
@@ -276,7 +276,8 @@ fn build_algorithm(
LlmTaskClassifier::new(classifier, weak, strong, *threshold).map_err(|error| {
ServerError::new(format!("llm_classifier route {route_name}: {error}"))
})?;
- let router = FallThrough::new(target_set).with_classifier(Arc::new(judge));
+ let router =
+ FallThrough::<State>::new_with_state(target_set).with_classifier(Arc::new(judge));
Ok(Arc::new(router))
}
}
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index f5d7e7d36..bb0ae635a 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -23,9 +23,7 @@ use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
use axum_server::tls_rustls::RustlsConfig;
-use libsy::{
- Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request, SharedState,
-};
+use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};
use serde_json::{json, Value};
use tokio::net::{TcpListener, TcpSocket};
use tracing::Level;
@@ -70,16 +68,14 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;
/// Shared server state used by all endpoint handlers.
#[derive(Clone)]
pub struct ServerState {
- /// For now making all the routes to be of type Algorithm<SharedState>
- /// Nachi's PR will help later
- routes: Arc<BTreeMap<String, Arc<dyn Algorithm<SharedState>>>>,
+ routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,
metrics: prometheus::Registry,
}
impl ServerState {
/// Creates server state from route model IDs and their libsy algorithms.
pub fn new(
- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm<SharedState>>)>,
+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,
) -> ServerResult<Self> {
let mut entries = BTreeMap::new();
for (model, algorithm) in routes {
@@ -106,7 +102,7 @@ impl ServerState {
self.routes.keys().map(String::as_str)
}
- fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm<SharedState>>> {
+ fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {
self.routes.get(model).map(Arc::clone)
}
}
@@ -361,7 +357,7 @@ fn resolve_route(
metadata: Metadata,
body: Value,
wire_format: WireFormat,
-) -> std::result::Result<(Arc<dyn Algorithm<SharedState>>, Request, String), Response> {
+) -> std::result::Result<(Arc<dyn Algorithm>, Request, String), Response> {
let llm_request = decode_request(wire_format, &body)
.map_err(|error| invalid_body_error(error.to_string()))?;
let requested_model = llm_request
@@ -403,10 +399,7 @@ async fn handle_llm_request(
Ok(resolved) => resolved,
Err(response) => return response,
};
- let (trace, response) = match algorithm
- .run(Context::<SharedState>::default(), request)
- .await
- {
+ let (trace, response) = match algorithm.run(Context::default(), request).await {
Ok(result) => result,
Err(error) => return algorithm_error(error),
};
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index c8bb644d1..e75fe5445 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -19,7 +19,7 @@ use axum::{Json, Router};
use http_body_util::BodyExt;
use libsy::algorithms::{FallThrough, Random};
use libsy::stage_router::{PickerMode, StageClassifier};
-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};
+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State as AlgorithmState};
use serde_json::{json, Value};
use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};
use switchyard_server::config::load_server_state;
@@ -157,8 +157,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
})
.collect(),
);
- let algorithm: Arc<dyn Algorithm<SharedState>> =
- Arc::new(Random::new(target_set, None, None)?);
+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);
Ok(((*route_model).to_string(), algorithm))
})
.collect::<TestResult<Vec<_>>>()?;
@@ -462,8 +461,9 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<S
llm_client: Some(weak),
},
]);
- let stage: Arc<dyn Algorithm<SharedState>> = Arc::new(
- FallThrough::new(targets).with_classifier(Arc::new(StageClassifier::new(mode, 0.5))),
+ let stage: Arc<dyn Algorithm> = Arc::new(
+ FallThrough::<AlgorithmState>::new_with_state(targets)
+ .with_classifier(Arc::new(StageClassifier::new(mode, 0.5))),
);
Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?)
}