Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(libsy): generalize fall-through state (#158)
nachiketb-nvidia61f41ffd4f91
diff --git a/crates/libsy/README.md b/crates/libsy/README.mdindex 9a0a6821f..2d1cc32bf 100644--- a/crates/libsy/README.md+++ b/crates/libsy/README.md@@ -11,7 +11,7 @@ so it drops into a proxy, gateway, or agent runtime.Build a target set, pick an algorithm, run a request:```rust-use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request, SharedState};+use switchyard_libsy::{Algorithm, Context, RoutedLlmClient, LlmTarget, LlmTargetSet, Request, State};use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};use switchyard_protocol::{completion_text, text_request};use std::sync::Arc;@@ -20,11 +20,13 @@ use std::sync::Arc;let client = Arc::new(MyClient { /* .. */ }) as Arc<dyn RoutedLlmClient>;let target = |name: &str| LlmTarget { semantic_name: name.into(), llm_client: Some(client.clone()) };-let targets = LlmTargetSet::new(vec![target("classifier"), target("strong"), target("weak")]);-let judge_target = targets.get_target("classifier")?;-let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(- FallThrough::new(targets).with_classifier(Arc::new(LlmTaskClassifier::new(- judge_target, "weak", "strong",+let classifier = target("classifier");+let strong = target("strong");+let weak = target("weak");+let targets = LlmTargetSet::new(vec![strong.clone(), weak.clone()]);+let algo: Arc<dyn Algorithm> = Arc::new(+ FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(LlmTaskClassifier::new(+ classifier, weak, strong, 0.5,)?)),);@@ -114,8 +116,7 @@ the id it calls — they can differ (`"strong"` → `"openai/gpt-4o"`) or coinci## Running a request-Hold the algorithm as `Arc<dyn Algorithm>` (or `Arc<dyn Algorithm<SharedState>>` for-`FallThrough`) and choose one of two entry points:+Hold the algorithm as `Arc<dyn Algorithm>` and choose one of two entry points:```rust// run: libsy drives the request to completion, serving each call with the target's@@ -225,8 +226,9 @@ Compose a classifier into `FallThrough`; the fall-through algorithm calls the juapplies its policy, and then invokes the selected target:```rust-let classifier = LlmTaskClassifier::new(judge_target, "weak", "strong")?;-let router = FallThrough::new(targets).with_classifier(Arc::new(classifier));+let classifier = LlmTaskClassifier::new(judge_target, weak_target, strong_target, 0.5)?;+let router =+ FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(classifier));```## Errorsdiff --git a/crates/libsy/examples/research_agent.rs b/crates/libsy/examples/research_agent.rsindex f9ff0f44d..0056ac02b 100644--- a/crates/libsy/examples/research_agent.rs+++ b/crates/libsy/examples/research_agent.rs@@ -17,7 +17,7 @@ use async_trait::async_trait;use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};use switchyard_libsy::{Algorithm, Context, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request,- Response, Result, RoutedLlmClient, SharedState,+ Response, Result, RoutedLlmClient, State,};use switchyard_protocol::{completion_text, text_request, text_response};@@ -64,7 +64,7 @@ fn targets() -> LlmTargetSet {}struct ResearchAgent {- algo: Arc<dyn Algorithm<SharedState>>,+ algo: Arc<dyn Algorithm>,}impl ResearchAgent {@@ -102,10 +102,10 @@ async fn main() -> Result<()> {let classifier = target_set.get_target(CLASSIFIER)?;let weak = target_set.get_target(WEAK)?;let strong = target_set.get_target(STRONG)?;- let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(- FallThrough::new(target_set).with_classifier(Arc::new(LlmTaskClassifier::new(- classifier, weak, strong, THRESHOLD,- )?)),+ let algo: Arc<dyn Algorithm> = Arc::new(+ FallThrough::<State>::new_with_state(target_set).with_classifier(Arc::new(+ LlmTaskClassifier::new(classifier, weak, strong, THRESHOLD)?,+ )),);let agent = ResearchAgent { algo };diff --git a/crates/libsy/examples/research_agent_core.rs b/crates/libsy/examples/research_agent_core.rsindex f0fc11ead..475cf468f 100644--- a/crates/libsy/examples/research_agent_core.rs+++ b/crates/libsy/examples/research_agent_core.rs@@ -14,7 +14,7 @@ use std::sync::Arc;use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};use switchyard_libsy::{Algorithm, Context, Decision, LibsyError, LlmResponse, LlmTarget, LlmTargetSet, Request,- Response, Result, SharedState, Step,+ Response, Result, State, Step,};use switchyard_protocol::{completion_text, text_request, text_response};use tokio_stream::StreamExt;@@ -51,7 +51,7 @@ fn targets() -> LlmTargetSet {}struct ResearchAgent {- algo: Arc<dyn Algorithm<SharedState>>,+ algo: Arc<dyn Algorithm>,}impl ResearchAgent {@@ -110,10 +110,10 @@ async fn main() -> Result<()> {let classifier = target_set.get_target(CLASSIFIER)?;let weak = target_set.get_target(WEAK)?;let strong = target_set.get_target(STRONG)?;- let algo: Arc<dyn Algorithm<SharedState>> = Arc::new(- FallThrough::new(target_set).with_classifier(Arc::new(LlmTaskClassifier::new(- classifier, weak, strong, THRESHOLD,- )?)),+ let algo: Arc<dyn Algorithm> = Arc::new(+ FallThrough::<State>::new_with_state(target_set).with_classifier(Arc::new(+ LlmTaskClassifier::new(classifier, weak, strong, THRESHOLD)?,+ )),);let mut agent = ResearchAgent { algo };diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rsindex 422a6865a..1d64e936f 100644--- a/crates/libsy/src/algorithms.rs+++ b/crates/libsy/src/algorithms.rs@@ -6,7 +6,7 @@//! Reach for them by name — `use switchyard_libsy::algorithms::Random` — rather than through the//! per-algorithm submodules.-pub mod fall_through;+mod fall_through;pub mod llm_class;pub mod noop;pub mod passthrough;@@ -16,7 +16,10 @@ pub use fall_through::{FallThrough, FallThroughDecision};pub use llm_class::LlmTaskClassifier;pub use noop::{Noop, NoopDecision};pub use passthrough::{Passthrough, PassthroughDecision};-pub use rand::{Random, RandomDecision};+pub use rand::{Random, RandomClassifier, RandomDecision};pub use util::{AffinityRouter, SubagentOverride};pub mod util;++#[cfg(test)]+mod subagent_affinity_tests;diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex b95f7b578..0152af477 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -1,40 +1,44 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Fall-through classifier routing: a stateful [`Algorithm`] that routes each turn+//! Fall-through classifier routing: a composable [`Algorithm`] that routes each turn//! through a processor chain and a classifier cascade.//!-//! Each turn: request-side [`Processor`]s fold facts into the session [`State`](crate::State); the+//! Each turn: request-side [`Processor`]s fold facts into the composition's state; the//! [`Classifier`] cascade is consulted in order and the first to score decides the target//! (its `argmax`); the [`Decision`] is published and then replayed to the processors so//! stateful ones (latch, affinity) can bind it.//!-//! [`FallThrough`] itself is stateless — one shared instance serves every session. The-//! per-session [`State`](crate::State) rides in the threaded [`Context<SharedState>`] the caller passes into-//! each turn, so routing can depend on accumulated history rather than the current request-//! alone. The state is held under a lock, so concurrent turns of the same session serialize-//! on it.+//! The default `FallThrough<()>` has no state. Stateful compositions share one private state+//! value across turns with the same session ID. Requests without a session ID use unretained+//! per-run state.-use std::sync::Arc;+use std::{collections::HashMap, sync::Arc};use async_trait::async_trait;+use parking_lot::Mutex;+use tokio::sync::Mutex as AsyncMutex;-use crate::core::{Classifier, Event, Processor, Score, SharedState};+use crate::core::{Classifier, Event, Processor, Score};use crate::{Algorithm, Context, Decision, Driver, LibsyError, LlmTargetSet, Request, Response, Result,RoutedLlmClient,};+type SessionStates<S> = Mutex<HashMap<String, Arc<AsyncMutex<S>>>>;+/// The decision a fall-through run produces: the selected model plus a human-readable reason.pub struct FallThroughDecision {- model: String,- reason: String,+ /// Target selected by the classifier cascade.+ pub selected_model: String,+ /// Human-readable explanation of the selection.+ pub reasoning: String,tier: Option<&'static str>,}impl Decision for FallThroughDecision {fn selected_model(&self) -> &str {- &self.model+ &self.selected_model}fn routing_tier(&self) -> Option<&str> {@@ -42,7 +46,7 @@ impl Decision for FallThroughDecision {}fn reasoning(&self) -> Option<&str> {- Some(&self.reason)+ Some(&self.reasoning)}fn as_any(&self) -> &dyn std::any::Any {@@ -52,36 +56,69 @@ impl Decision for FallThroughDecision {/// Processor chain → classifier cascade → routed model call. See the [module docs](self).///-/// Stateless: the per-session [`State`](crate::State) lives in the threaded [`Context<SharedState>`], so one-/// shared instance serves every session (see the module docs).-pub struct FallThrough {- processors: Vec<Arc<dyn Processor>>,- classifiers: Vec<Arc<dyn Classifier>>,+/// The generic state type is shared by every processor and classifier in the composition.+pub struct FallThrough<S = ()> {+ name: String,+ decision_reason: fn(&str, &Score) -> String,+ processors: Vec<Arc<dyn Processor<S>>>,+ classifiers: Vec<Arc<dyn Classifier<S>>>,targets: LlmTargetSet,+ session_states: Option<SessionStates<S>>,}-impl FallThrough {- /// Creates an empty router over `targets`; add components with the `with_*` builders.+impl FallThrough<()> {+ /// Creates an empty stateless router.pub fn new(targets: LlmTargetSet) -> Self {Self {+ name: "fall_through".to_string(),+ decision_reason: default_decision_reason,+ processors: Vec::new(),+ classifiers: Vec::new(),+ targets,+ session_states: None,+ }+ }+}++impl<S> FallThrough<S>+where+ S: Default + Send + 'static,+{+ /// Creates a router that retains one private `S` per session.+ pub fn new_with_state(targets: LlmTargetSet) -> Self {+ Self {+ name: "fall_through".to_string(),+ decision_reason: default_decision_reason,processors: Vec::new(),classifiers: Vec::new(),targets,+ session_states: Some(Mutex::new(HashMap::new())),}}+ /// Sets the stable, low-cardinality telemetry name for this composition.+ pub fn with_name(mut self, name: impl Into<String>) -> Self {+ self.name = name.into();+ self+ }++ /// Sets the decision reasoning for an algorithm assembled from this cascade.+ pub(crate) fn with_decision_reason(mut self, reason: fn(&str, &Score) -> String) -> Self {+ self.decision_reason = reason;+ self+ }+/// Appends a processor to the head-of-request chain.- pub fn with_processor(mut self, processor: Arc<dyn Processor>) -> Self {+ pub fn with_processor(mut self, processor: Arc<dyn Processor<S>>) -> Self {self.processors.push(processor);self}/// Appends a classifier to the cascade.- pub fn with_classifier(mut self, classifier: Arc<dyn Classifier>) -> Self {+ pub fn with_classifier(mut self, classifier: Arc<dyn Classifier<S>>) -> Self {self.classifiers.push(classifier);self}-/// Registers one dual-role component in *both* the processor chain and the classifier/// cascade.///@@ -90,41 +127,67 @@ impl FallThrough {/// shares that state through the instance, so both roles must be the same `Arc`./// Registering the two separately is easy to half-wire: omit the processor and the/// classifier silently never sees an assignment. This registers both at once.- pub fn with_component<T: Processor + Classifier + 'static>(self, component: Arc<T>) -> Self {+ pub fn with_component<T>(self, component: Arc<T>) -> Self+ where+ T: Processor<S> + Classifier<S> + 'static,+ {self.with_processor(component.clone()).with_classifier(component)}-}-#[async_trait]-impl Algorithm<SharedState> for FallThrough {- fn name(&self) -> &str {- "fall_through"- }- fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.targets.count_tokens_client()- }-- async fn create_run_task(- self: Arc<Self>,- ctx: Context<SharedState>,+ /// Executes the processor/classifier/target-call sequence for wrappers and the trait entrypoint.+ pub(crate) async fn execute(+ &self,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response> {- // Hold the session state (carried in the threaded context) for the whole- // request→decision fold, so a turn's fact accumulation is atomic and concurrent- // turns of the same session serialize on it.- let mut state = ctx.state.lock().await;-// The request is threaded mutably through the whole fold: any component may rewrite- // it, later components see the rewrite, and the final value is what reaches the model.+ // it, later components see the rewrite, and the final value reaches the model.let mut request = request;+ let session_state = self.session_state(&request);+ let (target, decision) = match session_state {+ Some(state) => {+ let mut state = state.lock().await;+ self.route(&mut state, &ctx, &driver, &mut request).await?+ }+ None => {+ let mut state = S::default();+ self.route(&mut state, &ctx, &driver, &mut request).await?+ }+ };++ driver+ .call_llm_target(ctx, &target, request, decision)+ .await+ }- // 1. Processor chain accumulates request-side facts into the session State.+ /// Returns this request's retained state without holding the registry lock.+ fn session_state(&self, request: &Request) -> Option<Arc<AsyncMutex<S>>> {+ let states = self.session_states.as_ref()?;+ let session_id = request+ .metadata+ .as_ref()?+ .session_id+ .as_deref()+ .filter(|session_id| !session_id.is_empty())?;+ let mut states = states.lock();+ let state = states+ .entry(session_id.to_string())+ .or_insert_with(|| Arc::new(AsyncMutex::new(S::default())));+ Some(Arc::clone(state))+ }++ async fn route(+ &self,+ state: &mut S,+ ctx: &Context,+ driver: &Driver,+ request: &mut Request,+ ) -> Result<(crate::LlmTarget, Arc<dyn Decision>)> {+ // 1. Processor chain accumulates request-side facts into the composition's state.for processor in &self.processors {- processor- .process(&mut state, Event::Request(&mut request))- .await?;+ processor.process(state, Event::Request(request)).await?;}// 2. Fall through the cascade: the first classifier to score decides (argmax). The@@ -132,9 +195,7 @@ impl Algorithm<SharedState> for FallThrough {let mut maybe_score: Option<Score> = None;let mut maybe_tier: Option<&'static str> = None;for classifier in &self.classifiers {- let scores = classifier- .score(&mut state, &mut request, Some(&driver))- .await?;+ let scores = classifier.score(state, request, Some(driver)).await?;maybe_score = scores.argmax(false)?;if let Some(s) = maybe_score.as_ref() {maybe_tier = classifier.routing_tier(&s.target);@@ -150,33 +211,63 @@ impl Algorithm<SharedState> for FallThrough {// 3. Resolve the target and publish the decision.let target = self.targets.get_target(&score.target)?;let decision: Arc<dyn Decision> = Arc::new(FallThroughDecision {- model: score.target.clone(),- reason: format!(- "fall-through selected {} (confidence {:.3})",- score.target, score.confidence- ),+ selected_model: score.target.clone(),+ reasoning: (self.decision_reason)(&self.name, &score),tier: maybe_tier,});- driver.info(ctx.without_state(), decision.clone()).await?;+ driver.info(ctx.clone(), decision.clone()).await?;- // 4. Replay the decision to the processors so stateful ones (latch / affinity) bind- // it into the session State.+ // 4. Replay the decision to the processors so stateful ones can bind it.for processor in &self.processors {processor- .process(&mut state, Event::Decision(decision.as_ref()))+ .process(+ state,+ Event::Decision {+ request,+ decision: decision.as_ref(),+ },+ ).await?;}- driver- .call_llm_target(ctx.without_state(), &target, request, decision)- .await+ Ok((target, decision))+ }+}++fn default_decision_reason(_name: &str, winner: &Score) -> String {+ format!(+ "fall-through selected {} (confidence {:.3})",+ winner.target, winner.confidence+ )+}++#[async_trait]+impl<S> Algorithm for FallThrough<S>+where+ S: Default + Send + 'static,+{+ fn name(&self) -> &str {+ &self.name+ }++ fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {+ self.targets.count_tokens_client()+ }++ async fn create_run_task(+ self: Arc<Self>,+ ctx: Context,+ driver: Driver,+ request: Request,+ ) -> Result<Response> {+ self.execute(ctx, driver, request).await}}#[cfg(test)]mod tests {use super::*;- use crate::core::{Classification, State};+ use crate::core::Classification;use switchyard_protocol::{completion_text, text_response, LlmRequest, Message, Metadata, Role,@@ -258,7 +349,7 @@ mod tests {impl Classifier for FixedClassifier {async fn score(&self,- _state: &mut State,+ _state: &mut (),_request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {@@ -300,13 +391,15 @@ mod tests {}}- /// Drives a shared router through one turn on the given session context, returning the- /// completion text + trace. Reuse the same `ctx` across calls to model a session.- async fn run_turn(- router: &Arc<FallThrough>,- ctx: Context<SharedState>,- ) -> Result<(String, Vec<Arc<dyn Decision>>)> {- let (trace, response) = router.clone().run(ctx, request()).await?;+ /// Drives a shared router with one request, returning the completion text + trace.+ async fn run_request<S>(+ router: &Arc<FallThrough<S>>,+ request: Request,+ ) -> Result<(String, Vec<Arc<dyn Decision>>)>+ where+ S: Default + Send + 'static,+ {+ let (trace, response) = router.clone().run(Context::default(), request).await?;let text = response.llm_response.into_agg()@@ -316,16 +409,24 @@ mod tests {Ok((text, trace))}- /// Drives a fresh router through one turn on a fresh session.+ /// Drives a shared router through one turn in the default test session.+ async fn run_turn<S>(router: &Arc<FallThrough<S>>) -> Result<(String, Vec<Arc<dyn Decision>>)>+ where+ S: Default + Send + 'static,+ {+ run_request(router, request()).await+ }++ /// Drives a fresh router through one turn.async fn run(router: FallThrough) -> Result<(String, Vec<Arc<dyn Decision>>)> {- run_turn(&Arc::new(router), Context::default()).await+ run_turn(&Arc::new(router)).await}// --- tests -------------------------------------------------------------------------#[tokio::test]async fn argmax_picks_the_highest_confidence_target() -> Result<()> {- let router = FallThrough::new(target_set(&["strong", "weak"]))+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![score("weak", 0.2), score("strong", 0.9)]));let (model, trace) = run(router).await?;assert_eq!(model, "strong");@@ -337,7 +438,7 @@ mod tests {#[tokio::test]async fn falls_through_the_first_abstaining_classifier() -> Result<()> {// First classifier abstains (empty); the second decides.- let router = FallThrough::new(target_set(&["strong", "weak"]))+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![])).with_classifier(fixed(vec![score("weak", 1.0)]));let (model, _) = run(router).await?;@@ -348,7 +449,7 @@ mod tests {#[tokio::test]async fn first_deciding_classifier_wins_the_cascade() -> Result<()> {// The first classifier decides; the second is never consulted.- let router = FallThrough::new(target_set(&["strong", "weak"]))+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![score("strong", 0.6)])).with_classifier(fixed(vec![score("weak", 1.0)]));let (model, _) = run(router).await?;@@ -359,7 +460,7 @@ mod tests {#[tokio::test]async fn all_abstaining_is_an_error() -> Result<()> {let router =- FallThrough::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![]));+ FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(fixed(vec![]));let error = run(router).await.err()@@ -381,7 +482,7 @@ mod tests {impl Classifier for NeedsDriver {async fn score(&self,- _state: &mut State,+ _state: &mut (),_request: &mut Request,driver: Option<&Driver>,) -> Result<Classification> {@@ -392,7 +493,7 @@ mod tests {}}- let router = FallThrough::new(target_set(&["strong", "weak"]))+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_classifier(Arc::new(NeedsDriver));let (model, _) = run(router).await?;assert_eq!(model, "strong");@@ -408,10 +509,10 @@ mod tests {#[async_trait]impl Processor for RecordingProcessor {- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {let kind = match event {Event::Request(_) => "request",- Event::Decision(_) => "decision",+ Event::Decision { .. } => "decision",_ => "other",};self.0.lock().push(kind);@@ -420,7 +521,7 @@ mod tests {}let seen = Arc::new(Mutex::new(Vec::new()));- let router = FallThrough::new(target_set(&["strong", "weak"]))+ let router = FallThrough::<()>::new(target_set(&["strong", "weak"])).with_processor(Arc::new(RecordingProcessor(seen.clone()))).with_classifier(fixed(vec![score("strong", 1.0)]));run(router).await?;@@ -438,7 +539,7 @@ mod tests {#[async_trait]impl Processor for Appender {- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {if let Event::Request(request) = event {request.llm_request@@ -457,7 +558,7 @@ mod tests {impl Classifier for TrailClassifier {async fn score(&self,- _state: &mut State,+ _state: &mut (),request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {@@ -486,7 +587,7 @@ mod tests {.with_processor(Arc::new(Appender("second"))).with_classifier(Arc::new(TrailClassifier(seen_by_classifier.clone())));- run_turn(&Arc::new(router), Context::default()).await?;+ run_turn(&Arc::new(router)).await?;// The classifier saw both processors' edits, in chain order, on top of the original.assert_eq!(*seen_by_classifier.lock(), vec!["hi", "first", "second"]);@@ -507,58 +608,72 @@ mod tests {}#[tokio::test]- async fn state_persists_across_turns() -> Result<()> {+ async fn state_is_shared_within_a_session_and_isolated_between_sessions() -> Result<()> {+ #[derive(Default)]+ struct TurnState {+ count: u32,+ }+// Increments the session turn count on every request.struct CountingProcessor;#[async_trait]- impl Processor for CountingProcessor {- async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {+ impl Processor<TurnState> for CountingProcessor {+ async fn process(&self, state: &mut TurnState, event: Event<'_>) -> Result<()> {if let Event::Request(_) = event {- state.turn_count += 1;+ state.count += 1;}Ok(())}}- // Routes to "weak" until the session has accumulated >= 2 turns, then "strong" —- // its decision depends only on state carried over from earlier turns.+ // Routes weak on a session's first turn and strong on later turns.struct ThresholdClassifier;#[async_trait]- impl Classifier for ThresholdClassifier {+ impl Classifier<TurnState> for ThresholdClassifier {async fn score(&self,- state: &mut State,+ state: &mut TurnState,_request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {- let target = if state.turn_count >= 2 {- "strong"- } else {- "weak"- };+ let target = if state.count >= 2 { "strong" } else { "weak" };Ok(Classification::Scores(vec![score(target, 1.0)]))}}- // One shared, stateless router; the session lives in the context we thread through.let router = Arc::new(- FallThrough::new(target_set(&["strong", "weak"]))+ FallThrough::<TurnState>::new_with_state(target_set(&["strong", "weak"])).with_processor(Arc::new(CountingProcessor)).with_classifier(Arc::new(ThresholdClassifier)),);- // One session context, reused across three turns. The turn counter accumulates in- // its `State`, so the classifier crosses its threshold on turn 2.- let ctx = Context::<SharedState>::default();- let (turn1, _) = run_turn(&router, ctx.clone()).await?;- let (turn2, _) = run_turn(&router, ctx.clone()).await?;- let (turn3, _) = run_turn(&router, ctx.clone()).await?;-- assert_eq!(turn1, "weak"); // count 1 — below threshold- assert_eq!(turn2, "strong"); // count 2 — state carried over from turn 1- assert_eq!(turn3, "strong"); // count 3 — still above threshold+ let (turn1, _) = run_turn(&router).await?;+ let (turn2, _) = run_turn(&router).await?;+ let (second_session, _) = run_request(+ &router,+ Request {+ metadata: Some(Metadata {+ session_id: Some("session-2".to_string()),+ ..Metadata::default()+ }),+ ..request()+ },+ )+ .await?;+ let anonymous = Request {+ metadata: None,+ ..request()+ };+ let (anonymous1, _) = run_request(&router, anonymous.clone()).await?;+ let (anonymous2, _) = run_request(&router, anonymous).await?;++ assert_eq!(turn1, "weak");+ assert_eq!(turn2, "strong");+ assert_eq!(second_session, "weak");+ assert_eq!(anonymous1, "weak");+ assert_eq!(anonymous2, "weak");Ok(())}}diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex eb5061bf3..a3dc63c57 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -230,7 +230,7 @@ impl LlmTaskClassifier {}#[async_trait]-impl Classifier for LlmTaskClassifier {+impl Classifier<State> for LlmTaskClassifier {fn routing_tier(&self, selected_model: &str) -> Option<&'static str> {if self.efficient_target == self.capable_target {None@@ -262,9 +262,7 @@ mod tests {use super::*;use switchyard_protocol::{completion_text, text_response, LlmClientError};- use crate::{- Algorithm, Context, LlmResponse, LlmTargetSet, Response, RoutedLlmClient, SharedState,- };+ use crate::{Algorithm, Context, LlmResponse, LlmTargetSet, Response, RoutedLlmClient};const TEST_THRESHOLD: f64 = 0.5;@@ -354,7 +352,7 @@ mod tests {}}- fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<super::super::FallThrough>> {+ fn router(client: Arc<dyn RoutedLlmClient>) -> Result<Arc<super::super::FallThrough<State>>> {let target = |name: &str| LlmTarget {semantic_name: name.to_string(),llm_client: Some(client.clone()),@@ -363,7 +361,7 @@ mod tests {let efficient = targets.get_target("efficient")?;let capable = targets.get_target("capable")?;Ok(Arc::new(- super::super::FallThrough::new(targets).with_classifier(Arc::new(+ super::super::FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(LlmTaskClassifier::new(target("judge"), efficient, capable, TEST_THRESHOLD)?,)),))@@ -384,9 +382,7 @@ mod tests {async fn an_unreachable_judge_routes_capable_instead_of_failing_the_request() -> Result<()> {let router = router(Arc::new(UnreachableJudgeClient))?;- let (trace, response) = router- .run(Context::<SharedState>::default(), classify_request())- .await?;+ let (trace, response) = router.run(Context::default(), classify_request()).await?;assert_eq!(trace.last().map(|d| d.selected_model()), Some("capable"));assert_eq!(@@ -402,14 +398,8 @@ mod tests {let router = router(client.clone())?;let request = classify_request;- router- .clone()- .run(Context::<SharedState>::default(), request())- .await?;- router- .clone()- .run(Context::<SharedState>::default(), request())- .await?;+ router.clone().run(Context::default(), request()).await?;+ router.clone().run(Context::default(), request()).await?;assert_eq!(client.calls(),diff --git a/crates/libsy/src/algorithms/noop.rs b/crates/libsy/src/algorithms/noop.rsindex 9b86746bd..389a1e72a 100644--- a/crates/libsy/src/algorithms/noop.rs+++ b/crates/libsy/src/algorithms/noop.rs@@ -34,17 +34,14 @@ impl Decision for NoopDecision {}#[async_trait::async_trait]-impl<S> Algorithm<S> for Noop-where- S: Clone + Send + Sync + 'static,-{+impl Algorithm for Noop {fn name(&self) -> &str {"noop"}async fn create_run_task(self: Arc<Self>,- ctx: Context<S>,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response> {@@ -55,7 +52,7 @@ wherelet decision: Arc<dyn Decision> = Arc::new(NoopDecision {model: model.clone(),});- driver.info(ctx.without_state(), decision.clone()).await?;+ driver.info(ctx, decision.clone()).await?;let llm_response = LlmResponse::Agg(AggLlmResponse {id: Some("switchyard-noop".to_string()),diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rsindex 4037cde68..35022ecd6 100644--- a/crates/libsy/src/algorithms/passthrough.rs+++ b/crates/libsy/src/algorithms/passthrough.rs@@ -40,10 +40,7 @@ impl Decision for PassthroughDecision {}#[async_trait::async_trait]-impl<S> Algorithm<S> for Passthrough-where- S: Clone + Send + Sync + 'static,-{+impl Algorithm for Passthrough {fn name(&self) -> &str {"passthrough"}@@ -58,14 +55,13 @@ whereasync fn create_run_task(self: Arc<Self>,- ctx: Context<S>,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response> {let decision: Arc<dyn Decision> = Arc::new(PassthroughDecision {model_id: self.target.semantic_name.clone(),});- let ctx = ctx.without_state();driver.info(ctx.clone(), decision.clone()).await?;driver.call_llm_target(ctx, &self.target, request, decision)diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rsindex 40f1abcec..60471a125 100644--- a/crates/libsy/src/algorithms/rand.rs+++ b/crates/libsy/src/algorithms/rand.rs@@ -1,55 +1,38 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Random router built on the [`Algorithm`] interfaces.+//! Random routing as a stateless [`FallThrough`] composition.//!-//! Selects one target from the set at random and calls it. Selection is uniform-//! by default and can use relative weights and a reproducible seed.+//! [`RandomClassifier`] selects one target; [`FallThrough`] owns the common+//! processor/classifier/target-call orchestration.use std::collections::BTreeSet;-use std::sync::{Arc, Mutex};+use std::sync::Arc;use async_trait::async_trait;+use parking_lot::Mutex;use rand::distributions::{Distribution, WeightedIndex};use rand::rngs::StdRng;use rand::SeedableRng;+use crate::algorithms::fall_through::{FallThrough, FallThroughDecision};use crate::{- Algorithm, Context, Decision, Driver, LibsyError, LlmTarget, LlmTargetSet, Request, Response,- Result, RoutedLlmClient,+ Algorithm, Classification, Classifier, Context, Driver, LibsyError, LlmTargetSet, Request,+ Response, Result, RoutedLlmClient, Score,};-/// Decision produced by [`Random`]: which target was chosen and why.-pub struct RandomDecision {- /// The randomly selected target/model.- pub selected_model: String,- /// Human-readable explanation of the choice.- pub reasoning: String,-}--impl Decision for RandomDecision {- fn selected_model(&self) -> &str {- &self.selected_model- }-- fn reasoning(&self) -> Option<&str> {- Some(&self.reasoning)- }+/// Compatibility name for the decision produced by [`Random`].+pub type RandomDecision = FallThroughDecision;- fn as_any(&self) -> &dyn std::any::Any {- self- }-}--/// Random router over a target set.-pub struct Random {- target_set: LlmTargetSet,+/// Stateless weighted classifier used by random fall-through routing.+pub struct RandomClassifier {+ targets: Vec<String>,distribution: WeightedIndex<f64>,rng: Mutex<StdRng>,}-impl Random {- /// Creates a router over `target_set`.+impl RandomClassifier {+ /// Creates a classifier over ordered target names.////// Missing weights default to one per target. Explicit weights are relative,/// follow target order, and need not sum to one. Zero disables a target.@@ -60,20 +43,12 @@ impl Random {/// Returns an error when targets are empty or duplicated, or when explicit/// weights have the wrong length, are negative or non-finite, or contain no/// positive value.- pub fn new(- target_set: LlmTargetSet,- weights: Option<Vec<f64>>,- seed: Option<u64>,- ) -> Result<Self> {- let target_count = target_set.targets().len();+ pub fn new(targets: Vec<String>, weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Self> {+ let target_count = targets.len();if target_count == 0 {return Err(LibsyError::NoTargets);}- let unique_targets = target_set- .targets()- .iter()- .map(|target| target.semantic_name.as_str())- .collect::<BTreeSet<_>>();+ let unique_targets = targets.iter().map(String::as_str).collect::<BTreeSet<_>>();if unique_targets.len() != target_count {return Err(LibsyError::AlgorithmError {message: "random targets must be unique".to_string(),@@ -107,19 +82,16 @@ impl Random {None => StdRng::from_entropy(),};Ok(Self {- target_set,+ targets,distribution,rng: Mutex::new(rng),})}- fn select_target(&self) -> Result<LlmTarget> {- let targets = self.target_set.targets();- let mut rng = self.rng.lock().map_err(|_| LibsyError::AlgorithmError {- message: "random number generator lock was poisoned".to_string(),- })?;+ fn select_target(&self) -> String {+ let mut rng = self.rng.lock();let index = self.distribution.sample(&mut *rng);- Ok(targets[index].clone())+ self.targets[index].clone()}}@@ -130,37 +102,74 @@ fn invalid_weights(message: String) -> LibsyError {}#[async_trait]-impl<S> Algorithm<S> for Random+impl<S> Classifier<S> for RandomClassifierwhere- S: Clone + Send + Sync + 'static,+ S: Send + 'static,{+ async fn score(+ &self,+ _state: &mut S,+ _request: &mut Request,+ _driver: Option<&Driver>,+ ) -> Result<Classification> {+ Ok(Classification::Scores(vec![Score {+ confidence: 1.0,+ target: self.select_target(),+ }]))+ }+}++/// Random router implemented as a stateless fall-through composition.+pub struct Random {+ inner: FallThrough<()>,+}++impl Random {+ /// Creates a router over `target_set`.+ ///+ /// # Errors+ ///+ /// Returns an error when targets or weights are invalid for [`RandomClassifier`].+ pub fn new(+ target_set: LlmTargetSet,+ weights: Option<Vec<f64>>,+ seed: Option<u64>,+ ) -> Result<Self> {+ let target_names = target_set+ .targets()+ .iter()+ .map(|target| target.semantic_name.clone())+ .collect();+ let classifier = Arc::new(RandomClassifier::new(target_names, weights, seed)?);+ let inner = FallThrough::<()>::new(target_set)+ .with_name("random")+ .with_decision_reason(random_decision_reason)+ .with_classifier(classifier);+ Ok(Self { inner })+ }+}++fn random_decision_reason(_name: &str, winner: &Score) -> String {+ format!("random routing selected target '{}'", winner.target)+}++#[async_trait]+impl Algorithm for Random {fn name(&self) -> &str {"random"}fn count_tokens_client(&self) -> Option<Arc<dyn RoutedLlmClient>> {- self.target_set.count_tokens_client()+ self.inner.count_tokens_client()}async fn create_run_task(self: Arc<Self>,- ctx: Context<S>,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response> {- let target = self.select_target()?;-- let selected = target.semantic_name.clone();- let decision: Arc<dyn Decision> = Arc::new(RandomDecision {- reasoning: format!("random routing selected target '{selected}'"),- selected_model: selected,- });-- let ctx = ctx.without_state();- driver.info(ctx.clone(), Arc::clone(&decision)).await?;- driver- .call_llm_target(ctx, &target, request, decision)- .await+ self.inner.execute(ctx, driver, request).await}}@@ -169,9 +178,10 @@ mod tests {use super::*;use std::collections::HashSet;- use switchyard_protocol::{completion_text, text_request, text_response};+ use switchyard_protocol::{completion_text, text_request, text_response, Metadata};- use crate::{LlmResponse, LlmTarget, Request, RoutedLlmClient, Signals};+ use crate::algorithms::AffinityRouter;+ use crate::{Decision, LlmResponse, LlmTarget, Request, RoutedLlmClient, Signals};/// Echoes the selected target so tests can inspect which target was called.struct EchoClient;@@ -199,8 +209,17 @@ mod tests {}}- /// Builds a random router whose targets all share an echo client.- fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {+ fn request_for_session(session_id: &str) -> Request {+ Request {+ metadata: Some(Metadata {+ session_id: Some(session_id.to_string()),+ ..Metadata::default()+ }),+ ..request()+ }+ }++ fn target_set(names: &[&str]) -> LlmTargetSet {let targets = names.iter().map(|name| LlmTarget {@@ -208,7 +227,12 @@ mod tests {llm_client: Some(Arc::new(EchoClient)),}).collect();- Random::new(LlmTargetSet::new(targets), weights, seed)+ LlmTargetSet::new(targets)+ }++ /// Builds a random router whose targets all share an echo client.+ fn algorithm(names: &[&str], weights: Option<Vec<f64>>, seed: Option<u64>) -> Result<Random> {+ Random::new(target_set(names), weights, seed)}fn shared_algorithm(names: &[&str]) -> Result<Arc<dyn Algorithm>> {@@ -322,6 +346,58 @@ mod tests {Ok(())}+ #[tokio::test]+ async fn affinity_reuses_the_initial_random_selection() -> Result<()> {+ let names = ["a/model", "b/model"];+ let affinity = Arc::new(AffinityRouter::new());+ let random = Arc::new(RandomClassifier::new(+ names.iter().map(|name| (*name).to_string()).collect(),+ None,+ Some(42),+ )?);+ let algorithm: Arc<dyn Algorithm> = Arc::new(+ FallThrough::<()>::new(target_set(&names))+ .with_name("affinity_random")+ .with_processor(affinity.clone())+ .with_classifier(affinity.clone())+ .with_classifier(random),+ );++ let (_, first) = algorithm+ .clone()+ .run(Context::default(), request_for_session("session-1"))+ .await?;+ let selected = first+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default();++ let mut state = ();+ let mut request = request_for_session("session-1");+ let retained = affinity+ .score(&mut state, &mut request, None)+ .await?+ .argmax(false)?;+ assert_eq!(+ retained.map(|score| score.target),+ Some(selected.to_string())+ );++ let (_, second) = algorithm+ .run(Context::default(), request_for_session("session-1"))+ .await?;+ assert_eq!(+ second+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ selected+ );+ Ok(())+ }+#[test]fn rejects_invalid_weights() {let cases = [diff --git a/crates/libsy/tests/subagent_affinity.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rssimilarity index 73%rename from crates/libsy/tests/subagent_affinity.rsrename to crates/libsy/src/algorithms/subagent_affinity_tests.rsindex fca6403d7..ba7cd1d65 100644--- a/crates/libsy/tests/subagent_affinity.rs+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs@@ -1,22 +1,22 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-//! Integration tests for composing the sub-agent override with affinity routing.+//! Tests for composing the sub-agent override with affinity routing.//!//! The two are independent classifiers in one [`FallThrough`] cascade: the override decides//! *which* target delegated work belongs on, affinity decides *how long* that decision-//! lives. These tests drive them through the crate's public API, so they also pin that the-//! pieces needed to compose a cascade are actually exported.+//! lives.use std::collections::BTreeMap;use std::sync::Arc;use async_trait::async_trait;-use switchyard_libsy::algorithms::{AffinityRouter, FallThrough, SubagentOverride};-use switchyard_libsy::{+use super::fall_through::FallThrough;+use super::{AffinityRouter, SubagentOverride};+use crate::{Algorithm, Classification, Classifier, Context, Decision, Driver, LlmResponse, LlmTarget,- LlmTargetSet, Metadata, Request, Response, Result, RoutedLlmClient, Score, SharedState, State,+ LlmTargetSet, Metadata, Request, Response, Result, RoutedLlmClient, Score,};use switchyard_protocol::{completion_text, text_request, text_response};@@ -45,7 +45,7 @@ struct AlwaysOrchestrator;impl Classifier for AlwaysOrchestrator {async fn score(&self,- _state: &mut State,+ _state: &mut (),_request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {@@ -85,24 +85,19 @@ fn request(headers: &[(&str, &str)]) -> Request {/// terminal classifier serves everything else.fn router() -> Arc<FallThrough> {Arc::new(- FallThrough::new(targets())+ FallThrough::<()>::new(targets()).with_component(Arc::new(AffinityRouter::for_subagents())).with_classifier(Arc::new(SubagentOverride::new("worker"))).with_classifier(Arc::new(AlwaysOrchestrator)),)}-/// Runs one turn on `ctx`, returning the target that served it.-///-/// `ctx` carries the per-session [`State`] the processor chain folds into. The affinity-/// assignments are held on the `AffinityRouter` instance instead, so replay follows the-/// shared router rather than the context.-async fn turn(- router: &Arc<FallThrough>,- ctx: Context<SharedState>,- headers: &[(&str, &str)],-) -> Result<String> {- let (_, response) = router.clone().run(ctx, request(headers)).await?;+/// Runs one turn, returning the target that served it.+async fn turn(router: &Arc<FallThrough>, headers: &[(&str, &str)]) -> Result<String> {+ let (_, response) = router+ .clone()+ .run(Context::default(), request(headers))+ .await?;Ok(response.llm_response.as_agg()@@ -120,53 +115,42 @@ fn child(agent: &str) -> Vec<(&str, &str)> {#[tokio::test]async fn root_traffic_falls_through_to_the_terminal_classifier() -> Result<()> {- let ctx = Context::<SharedState>::default();// No sub-agent lineage: affinity abstains (subagents only), the override abstains, and// the terminal classifier serves the turn.- let served = turn(&router(), ctx, &[("x-claude-code-session-id", "session-1")]).await?;+ let served = turn(&router(), &[("x-claude-code-session-id", "session-1")]).await?;assert_eq!(served, "orchestrator");Ok(())}#[tokio::test]async fn delegated_work_is_routed_to_the_worker() -> Result<()> {- let ctx = Context::<SharedState>::default();- assert_eq!(turn(&router(), ctx, &child("child-1")).await?, "worker");+ assert_eq!(turn(&router(), &child("child-1")).await?, "worker");Ok(())}#[tokio::test]async fn the_override_seeds_a_pin_that_affinity_replays() -> Result<()> {let router = router();- let ctx = Context::<SharedState>::default();// Turn 1: affinity has no assignment, so the override decides and the decision is// replayed into affinity.- assert_eq!(- turn(&router, ctx.clone(), &child("child-1")).await?,- "worker"- );+ assert_eq!(turn(&router, &child("child-1")).await?, "worker");// Turns 2-3: the lineage header still identifies the child, but affinity — not the// override — is now the classifier that answers, because it scores first.- assert_eq!(- turn(&router, ctx.clone(), &child("child-1")).await?,- "worker"- );- assert_eq!(turn(&router, ctx, &child("child-1")).await?, "worker");+ assert_eq!(turn(&router, &child("child-1")).await?, "worker");+ assert_eq!(turn(&router, &child("child-1")).await?, "worker");Ok(())}#[tokio::test]async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()> {let router = router();- let ctx = Context::<SharedState>::default();// A Codex `compact` turn is sub-agent *lineage* but not delegated *work*: the two// predicates differ on purpose, so the override abstains and it routes normally.let served = turn(&router,- ctx,&[("x-codex-session-id", "session-1"),("x-openai-subagent", "compact"),@@ -180,7 +164,7 @@ async fn harness_maintenance_turns_are_not_forced_to_the_worker() -> Result<()>/// Builds a cascade whose override scores `worker`, sharing `affinity` across instances.fn router_overriding_to(affinity: Arc<AffinityRouter>, worker: &str) -> Arc<FallThrough> {Arc::new(- FallThrough::new(targets())+ FallThrough::<()>::new(targets()).with_component(affinity).with_classifier(Arc::new(SubagentOverride::new(worker))).with_classifier(Arc::new(AlwaysOrchestrator)),@@ -197,14 +181,8 @@ async fn the_pin_outlives_the_policy_that_seeded_it() -> Result<()> {let seed = router_overriding_to(affinity.clone(), "worker");let rebound = router_overriding_to(affinity, "reviewer");- assert_eq!(- turn(&seed, Context::default(), &child("child-1")).await?,- "worker"- );- assert_eq!(- turn(&rebound, Context::default(), &child("child-1")).await?,- "worker"- );+ assert_eq!(turn(&seed, &child("child-1")).await?, "worker");+ assert_eq!(turn(&rebound, &child("child-1")).await?, "worker");Ok(())}@@ -216,14 +194,8 @@ async fn without_a_shared_pin_the_second_policy_wins() -> Result<()> {let seed = router_overriding_to(Arc::new(AffinityRouter::for_subagents()), "worker");let rebound = router_overriding_to(Arc::new(AffinityRouter::for_subagents()), "reviewer");- assert_eq!(- turn(&seed, Context::default(), &child("child-1")).await?,- "worker"- );- assert_eq!(- turn(&rebound, Context::default(), &child("child-1")).await?,- "reviewer"- );+ assert_eq!(turn(&seed, &child("child-1")).await?, "worker");+ assert_eq!(turn(&rebound, &child("child-1")).await?, "reviewer");Ok(())}@@ -237,20 +209,11 @@ async fn distinct_children_are_pinned_independently() -> Result<()> {let seed = router_overriding_to(affinity.clone(), "worker");let sibling = router_overriding_to(affinity, "reviewer");- assert_eq!(- turn(&seed, Context::default(), &child("child-1")).await?,- "worker"- );- assert_eq!(- turn(&sibling, Context::default(), &child("child-2")).await?,- "reviewer"- );+ assert_eq!(turn(&seed, &child("child-1")).await?, "worker");+ assert_eq!(turn(&sibling, &child("child-2")).await?, "reviewer");// child-1's own pin is untouched by the sibling's assignment: affinity replays it// even through the cascade whose override would otherwise score "reviewer".- assert_eq!(- turn(&sibling, Context::default(), &child("child-1")).await?,- "worker"- );+ assert_eq!(turn(&sibling, &child("child-1")).await?, "worker");Ok(())}@@ -259,11 +222,10 @@ async fn a_cascade_without_the_override_still_routes_root_traffic() -> Result<()// Affinity and the override are independent: dropping the override leaves a valid// cascade, which is the point of composing them rather than nesting one in the other.let router = Arc::new(- FallThrough::new(targets())+ FallThrough::<()>::new(targets()).with_component(Arc::new(AffinityRouter::for_subagents())).with_classifier(Arc::new(AlwaysOrchestrator)),);- let ctx = Context::<SharedState>::default();- assert_eq!(turn(&router, ctx, &child("child-1")).await?, "orchestrator");+ assert_eq!(turn(&router, &child("child-1")).await?, "orchestrator");Ok(())}diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rsindex 5d809645c..8f33db35a 100644--- a/crates/libsy/src/algorithms/util/affinity.rs+++ b/crates/libsy/src/algorithms/util/affinity.rs@@ -7,9 +7,8 @@//! forces that model on later requests sharing the identity. It is one object that plays//! both SDK roles, so registering it as a processor and a classifier cannot drift apart://!-//! - As a [`Processor`] it *writes* the assignment: it captures the request's identity on-//! [`Event::Request`] and binds it to the chosen model on [`Event::Decision`], the first-//! assignment for an identity winning.+//! - As a [`Processor`] it *writes* the assignment: [`Event::Decision`] carries the request+//! whose chosen model is retained, with the first assignment for an identity winning.//! - As a [`Classifier`] it *reads* the assignment: it scores the retained model with//! confidence `1.0`, or returns no scores to abstain.//!@@ -24,7 +23,7 @@ use async_trait::async_trait;use parking_lot::Mutex;use switchyard_protocol::Request;-use crate::{Classification, Classifier, Event, Processor, Score, State};+use crate::{Classification, Classifier, Event, Processor, Score};/// Upper bound on retained assignments, keeping the process-local map from growing/// without limit; the oldest entry is evicted once the bound is reached.@@ -42,23 +41,12 @@ enum AffinityKey {Subagent { session: String, agent: String },}-/// The affinity memory: the retained model per identity, plus the key captured from the-/// current turn's request while its decision is pending.-#[derive(Default)]-struct AffinityAssignments {- /// Identity → retained model, first assignment winning.- assignments: HashMap<AffinityKey, String>,- /// Key captured on this turn's [`Event::Request`], consumed by its [`Event::Decision`].- pending: Option<AffinityKey>,-}-/// Retains a model per request identity and forces it on later matching requests.////// Register the same instance as both a processor and a classifier; the two roles share-/// the retained assignments through this instance's interior storage. Because a decision-/// event carries only the chosen model, the request's identity is captured first (on-/// [`Event::Request`]) and consumed when the decision arrives — this assumes a turn's-/// request is observed before its decision, as the driving algorithm folds a turn in order.+/// the retained assignments through this instance's interior storage. Decision events carry+/// their originating request, so concurrent turns cannot bind one request's identity to+/// another request's selected model.////// [`with_latch_only`](Self::with_latch_only) narrows *which* models are retained — a/// decision for any other model routes normally but is not latched (the escalation latch:@@ -71,10 +59,9 @@ pub struct AffinityRouter {subagents_only: bool,/// Retained assignments, shared across this router's processor and classifier roles.///- /// Held on the instance rather than in [`State`] so the two roles share one- /// process-local map through a single registered [`Arc`](std::sync::Arc); bounded by- /// [`MAX_ASSIGNMENTS`].- assignments: Mutex<AffinityAssignments>,+ /// Held on the instance so the two roles share one process-local map through a+ /// single registered [`Arc`](std::sync::Arc); bounded by [`MAX_ASSIGNMENTS`].+ assignments: Mutex<HashMap<AffinityKey, String>>,}impl AffinityRouter {@@ -125,45 +112,40 @@ impl AffinityRouter {}#[async_trait]-impl Processor for AffinityRouter {- async fn process(&self, _state: &mut State, event: Event<'_>) -> crate::Result<()> {- // No `.await` is held across this guard.- let mut fact = self.assignments.lock();- match event {- // Capture the identity now; the model it maps to is only known at the decision.- Event::Request(request) => {- fact.pending = self.affinity_key(request);- }- // Bind the captured identity to the chosen model, keeping any earlier winner.- Event::Decision(decision) => {+impl<S> Processor<S> for AffinityRouter+where+ S: Send + 'static,+{+ async fn process(&self, _state: &mut S, event: Event<'_>) -> crate::Result<()> {+ if let Event::Decision { request, decision } = event {+ if let Some(key) = self.affinity_key(request) {let model = decision.selected_model();- let Some(key) = fact.pending.take() else {- return Ok(());- };- // Latch only permitted models; others consume `pending` but are not retained.- if self.should_latch(model) && !fact.assignments.contains_key(&key) {- evict_if_full(&mut fact.assignments);- fact.assignments.insert(key, model.to_string());+ let mut assignments = self.assignments.lock();+ if self.should_latch(model) && !assignments.contains_key(&key) {+ evict_if_full(&mut assignments);+ assignments.insert(key, model.to_string());}}- _ => {}}Ok(())}}#[async_trait]-impl Classifier for AffinityRouter {+impl<S> Classifier<S> for AffinityRouter+where+ S: Send + 'static,+{async fn score(&self,- _state: &mut State,+ _state: &mut S,request: &mut Request,_driver: Option<&crate::Driver>,) -> crate::Result<Classification> {let Some(key) = self.affinity_key(request) else {return Ok(Classification::Scores(Vec::new()));};- let assigned = self.assignments.lock().assignments.get(&key).cloned();+ let assigned = self.assignments.lock().get(&key).cloned();Ok(Classification::Scores(match assigned {Some(target) => vec![Score {confidence: 1.0,@@ -189,10 +171,9 @@ mod tests {use std::sync::Arc;- use switchyard_protocol::{text_request, Decision, Metadata, Signals};+ use switchyard_protocol::{text_request, Decision, Metadata};- /// Boxed, thread-safe error type keeping the test helpers ergonomic; libsy errors- /// convert into it through `?`.+ /// Boxed, thread-safe error type keeping the test helpers ergonomic.type BoxErr = Box<dyn std::error::Error + Send + Sync>;/// A decision that reports a fixed selected model.@@ -241,13 +222,18 @@ mod tests {/// Folds a request and its decision through the router, retaining `model`.async fn retain(router: &AffinityRouter,- state: &mut State,+ state: &mut (),request: &mut Request,model: &'static str,) -> Result<(), BoxErr> {- router.process(state, Event::Request(request)).await?;router- .process(state, Event::Decision(&FixedDecision(model)))+ .process(+ state,+ Event::Decision {+ request,+ decision: &FixedDecision(model),+ },+ ).await?;Ok(())}@@ -255,7 +241,7 @@ mod tests {/// Scores through the definitive classification variant used by affinity.async fn scores(classifier: &dyn Classifier,- state: &mut State,+ state: &mut (),request: &mut Request,) -> Result<Vec<Score>, BoxErr> {match classifier.score(state, request, None).await? {@@ -267,7 +253,7 @@ mod tests {#[tokio::test]async fn session_retains_first_model_across_requests() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();let mut first = request(session("session-1", "agent-a"));retain(&router, &mut state, &mut first, "model-a").await?;@@ -284,7 +270,7 @@ mod tests {#[tokio::test]async fn subagent_only_retains_children_without_latching_root_traffic() -> Result<(), BoxErr> {let router = AffinityRouter::for_subagents();- let mut state = State::default();+ let mut state = ();let mut root = request(session("session-1", "root-agent"));retain(&router, &mut state, &mut root, "model-a").await?;@@ -304,7 +290,7 @@ mod tests {#[tokio::test]async fn first_decision_wins() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();let mut req = request(session("session-1", "agent-a"));retain(&router, &mut state, &mut req, "model-a").await?;@@ -322,7 +308,7 @@ mod tests {#[tokio::test]async fn subagent_is_keyed_by_agent_not_task() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();let mut first = request(subagent("child-1", "task-1"));retain(&router, &mut state, &mut first, "model-a").await?;@@ -340,7 +326,7 @@ mod tests {#[tokio::test]async fn distinct_subagents_are_assigned_independently() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();// One child in the session is pinned...retain(@@ -360,7 +346,7 @@ mod tests {#[tokio::test]async fn subagent_does_not_inherit_session_assignment() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();// The session root is pinned, but a sub-agent is keyed separately...retain(@@ -380,7 +366,7 @@ mod tests {#[tokio::test]async fn classifier_abstains_without_a_session() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();// No session id at all: nothing to key on.let mut req = request(Metadata::default());@@ -395,14 +381,17 @@ mod tests {let router = Arc::new(AffinityRouter::new());let processor: Arc<dyn Processor> = router.clone();let classifier: Arc<dyn Classifier> = router;- let mut state = State::default();+ let mut state = ();- let mut first = request(session("session-1", "agent-a"));+ let first = request(session("session-1", "agent-a"));processor- .process(&mut state, Event::Request(&mut first))- .await?;- processor- .process(&mut state, Event::Decision(&FixedDecision("model-a")))+ .process(+ &mut state,+ Event::Decision {+ request: &first,+ decision: &FixedDecision("model-a"),+ },+ ).await?;let mut second = request(session("session-1", "agent-b"));@@ -415,13 +404,19 @@ mod tests {}#[tokio::test]- async fn decision_without_a_request_is_ignored() -> Result<(), BoxErr> {+ async fn decision_without_an_affinity_identity_is_ignored() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();+ let unkeyed = request(Metadata::default());- // A decision with no captured identity (no prior request) stores nothing.router- .process(&mut state, Event::Decision(&FixedDecision("model-a")))+ .process(+ &mut state,+ Event::Decision {+ request: &unkeyed,+ decision: &FixedDecision("model-a"),+ },+ ).await?;let mut req = request(session("session-1", "agent-a"));@@ -430,33 +425,50 @@ mod tests {}#[tokio::test]- async fn unrelated_events_preserve_the_pending_identity() -> Result<(), BoxErr> {+ async fn decisions_retain_their_originating_request_identity() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();+ let mut first = request(session("session-1", "agent-a"));+ let mut second = request(session("session-2", "agent-b"));- let mut req = request(session("session-1", "agent-a"));- router.process(&mut state, Event::Request(&mut req)).await?;- // A stray signal between the request and its decision must not drop the captured- // identity, nor create an assignment of its own.+ // Replay decisions in the opposite order. Each decision carries its request,+ // so the assignments cannot cross.router- .process(&mut state, Event::Signal(&Signals {}))+ .process(+ &mut state,+ Event::Decision {+ request: &second,+ decision: &FixedDecision("model-b"),+ },+ ).await?;router- .process(&mut state, Event::Decision(&FixedDecision("model-a")))+ .process(+ &mut state,+ Event::Decision {+ request: &first,+ decision: &FixedDecision("model-a"),+ },+ ).await?;- let scores = scores(&router, &mut state, &mut req).await?;+ let first_scores = scores(&router, &mut state, &mut first).await?;+ let second_scores = scores(&router, &mut state, &mut second).await?;assert_eq!(- scores.first().map(|score| score.target.as_str()),+ first_scores.first().map(|score| score.target.as_str()),Some("model-a"));+ assert_eq!(+ second_scores.first().map(|score| score.target.as_str()),+ Some("model-b")+ );Ok(())}#[tokio::test]async fn distinct_sessions_are_assigned_independently() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();retain(&router,@@ -499,7 +511,7 @@ mod tests {#[tokio::test]async fn subagent_without_an_agent_id_is_not_keyed() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();// The sub-agent flag is set but no agent id is present, so no key can be formed;// the request is neither retained nor scored.@@ -517,7 +529,7 @@ mod tests {#[tokio::test]async fn assignments_are_bounded_by_the_cap() -> Result<(), BoxErr> {let router = AffinityRouter::new();- let mut state = State::default();+ let mut state = ();// One distinct session past the cap forces exactly one eviction.for index in 0..=MAX_ASSIGNMENTS {@@ -531,7 +543,7 @@ mod tests {.await?;}- let len = router.assignments.lock().assignments.len();+ let len = router.assignments.lock().len();assert_eq!(len, MAX_ASSIGNMENTS);Ok(())}@@ -539,7 +551,7 @@ mod tests {#[tokio::test]async fn latch_only_retains_matching_models() -> Result<(), BoxErr> {let router = AffinityRouter::new().with_latch_only(["strong"]);- let mut state = State::default();+ let mut state = ();let mut req = request(session("session-1", "agent-a"));// A "weak" decision is not retained — a later turn is not latched.diff --git a/crates/libsy/src/algorithms/util/handoff_notes.rs b/crates/libsy/src/algorithms/util/handoff_notes.rsindex dbaba4c67..bbf9ede02 100644--- a/crates/libsy/src/algorithms/util/handoff_notes.rs+++ b/crates/libsy/src/algorithms/util/handoff_notes.rs@@ -67,9 +67,9 @@ impl HandoffNoteProcessor {}#[async_trait]-impl Processor for HandoffNoteProcessor {+impl Processor<State> for HandoffNoteProcessor {async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {- let Event::Decision(decision) = event else {+ let Event::Decision { decision, .. } = event else {return Ok(());};let tier = decision.selected_model().to_string();@@ -90,7 +90,7 @@ impl Processor for HandoffNoteProcessor {#[cfg(test)]mod tests {use super::*;- use crate::Decision;+ use crate::{Decision, Request};struct FakeDecision(&'static str);impl Decision for FakeDecision {@@ -115,8 +115,16 @@ mod tests {}async fn run(processor: &HandoffNoteProcessor, state: &mut State, tier: &'static str) {+ let request = Request::default();+ let decision = FakeDecision(tier);processor- .process(state, Event::Decision(&FakeDecision(tier)))+ .process(+ state,+ Event::Decision {+ request: &request,+ decision: &decision,+ },+ ).await.unwrap();}diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 04b29cdd8..e876debe4 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -118,7 +118,7 @@ where}#[async_trait]-impl<J, P> Classifier for JudgeClassifier<J, P>+impl<J, P> Classifier<State> for JudgeClassifier<J, P>whereJ: Judge,P: JudgePolicy<Verdict = J::Verdict>,@@ -381,8 +381,9 @@ mod tests {#[tokio::test]async fn a_missing_driver_is_an_error_not_a_fallback() -> Result<()> {+ let mut request = request();let error = classifier()- .score(&mut State::default(), &mut request(), None)+ .score(&mut State::default(), &mut request, None).await.err().ok_or_else(|| LibsyError::AlgorithmError {diff --git a/crates/libsy/src/algorithms/util/stage_router.rs b/crates/libsy/src/algorithms/util/stage_router.rsindex c8edd4062..a9d69251c 100644--- a/crates/libsy/src/algorithms/util/stage_router.rs+++ b/crates/libsy/src/algorithms/util/stage_router.rs@@ -305,7 +305,7 @@ impl StageClassifier {}#[async_trait]-impl Classifier for StageClassifier {+impl Classifier<State> for StageClassifier {async fn score(&self,state: &mut State,diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rsindex aa3b4e598..166c8f121 100644--- a/crates/libsy/src/algorithms/util/subagent.rs+++ b/crates/libsy/src/algorithms/util/subagent.rs@@ -7,8 +7,8 @@//! sub-agent work ([`Metadata::is_subagent_work`]) and abstains for everything else, so a//! cascade falls through to its later classifiers on ordinary traffic.//!-//! It is stateless and holds only the worker's *name*: the-//! [`FallThrough`](crate::algorithms::FallThrough) cascade resolves that name against its+//! It is stateless and holds only the worker's *name*: the fall-through cascade resolves it+//! against its//! target set. Keeping the policy independent of any memory of past decisions is what lets//! it compose with a stateful classifier such as//! [`AffinityRouter`](crate::algorithms::AffinityRouter) — the override decides *which*@@ -17,7 +17,7 @@use async_trait::async_trait;-use crate::{Classification, Classifier, Driver, Metadata, Request, Result, Score, State};+use crate::{Classification, Classifier, Driver, Metadata, Request, Result, Score};/// Scores a fixed worker target for delegated sub-agent work; abstains otherwise.pub struct SubagentOverride {@@ -38,10 +38,13 @@ impl SubagentOverride {}#[async_trait]-impl Classifier for SubagentOverride {+impl<S> Classifier<S> for SubagentOverride+where+ S: Send + 'static,+{async fn score(&self,- _state: &mut State,+ _state: &mut S,request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {@@ -87,7 +90,7 @@ mod tests {/// Scores `headers` through the override, returning the winning target if it scored.async fn selected(headers: &[(&str, &str)]) -> Result<Option<String>> {- let mut state = State::default();+ let mut state = ();let classification = SubagentOverride::new("worker").score(&mut state, &mut request(headers), None).await?;@@ -135,7 +138,7 @@ mod tests {async fn delegated_work_is_scored_definitively() -> Result<()> {// Confidence 1.0 under `Scores` (never `Ambiguous`), so the cascade stops here// rather than consulting later classifiers.- let mut state = State::default();+ let mut state = ();let classification = SubagentOverride::new("worker").score(&mut state,diff --git a/crates/libsy/src/algorithms/util/tool_signals.rs b/crates/libsy/src/algorithms/util/tool_signals.rsindex 143d1044c..1b0a0d9f2 100644--- a/crates/libsy/src/algorithms/util/tool_signals.rs+++ b/crates/libsy/src/algorithms/util/tool_signals.rs@@ -274,7 +274,7 @@ impl Default for ToolSignalProcessor {}#[async_trait]-impl Processor for ToolSignalProcessor {+impl Processor<State> for ToolSignalProcessor {async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {if let Event::Request(req) = event {let tool_signal = ToolSignals::from_request(req, Some(self.recent_window));diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex b3e71bc4e..0d329601a 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -327,19 +327,9 @@ impl LlmTargetSet {/// or [`run_stream`](Self::run_stream) (drive the [`Step`] stream yourself).////// Methods take `self: Arc<Self>`: one algorithm (`Arc<dyn Algorithm>`) is shared across-/// requests and run concurrently, so it owns its thread-safety. Stateless algorithms-/// (the reference routers) get this for free; a stateful one keeps its per-session state-/// in the threaded [`Context<S>`], not in the shared algorithm.-///-/// The `S` type parameter is the per-session state carried in `Context<S>`. It defaults-/// to `()`, so stateless algorithms just write `impl Algorithm for MyRouter` and callers-/// keep using `Arc<dyn Algorithm>`. A stateful algorithm picks its own state type (e.g.-/// `impl Algorithm<SharedState> for FallThrough`).+/// requests and run concurrently, so it owns its thread-safety and any shared state.#[async_trait]-pub trait Algorithm<S = ()>: Send + Sync + 'static-where- S: Clone + Send + Sync + 'static,-{+pub trait Algorithm: Send + Sync + 'static {/// Stable, low-cardinality name identifying this algorithm — the/// `algorithm` attribute on every span, metric, and log line the crate/// emits for its runs (see the crate docs' Observability section).@@ -349,10 +339,10 @@ where/// publish [`Decision`]s with [`Driver::info`], and return the final [`Response`]./// The method an algorithm implements; [`run`](Self::run) / [`run_stream`](Self::run_stream)/// drive it. `ctx` carries the request's cross-cutting values (today: the- /// algorithm's telemetry label in [`Context::values`]) and per-session `state`.+ /// algorithm's telemetry label in [`Context::values`]).async fn create_run_task(self: Arc<Self>,- ctx: Context<S>,+ ctx: Context,driver: Driver,request: Request,) -> Result<Response>;@@ -401,7 +391,7 @@ where/// Process a request to completion, returning a stream of [`Step`]s./// Each [`Step::CallLlm`] is an offloaded model call the consumer must serve./// The stream ends with a [`Step::ReturnToAgent`] on success, or an `Err` item on failure.- fn run_stream(self: Arc<Self>, ctx: Context<S>, request: Request) -> StepStream {+ fn run_stream(self: Arc<Self>, ctx: Context, request: Request) -> StepStream {// Stamp the algorithm's telemetry label into the request context; the// context rides on every driver call, so its telemetry is attributed.let mut ctx = ctx;@@ -431,8 +421,7 @@ wherelet abort_guard = AbortOnDrop(handle.abort_handle());let finish_driver = driver.clone();- // The terminal step carries no session state; hand `finish` the base context.- let finish_ctx = ctx.without_state();+ let finish_ctx = ctx;let tail: StepStream = Box::pin(futures::stream::once(async move {let result = match handle.await {@@ -456,7 +445,7 @@ where/// [`Decision`]s the algorithm made along the way.async fn run(self: Arc<Self>,- ctx: Context<S>,+ ctx: Context,request: Request,) -> Result<(Vec<Arc<dyn Decision>>, Response)> {// Serve one offloaded call with its target's default client. A failed *model*diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rsindex 0d949ddb1..2ab90f6a0 100644--- a/crates/libsy/src/core/classifier.rs+++ b/crates/libsy/src/core/classifier.rs@@ -1,7 +1,7 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-use crate::{Driver, LibsyError, Result, State};+use crate::{Driver, LibsyError, Result};use async_trait::async_trait;use switchyard_protocol::Request;@@ -67,9 +67,9 @@ fn argmax(scores: &[Score]) -> Result<Option<Score>> {Ok(best.cloned())}-/// Scores each of the classifier's targets given State the current Request+/// Scores targets from the current request and the composition's state.#[async_trait]-pub trait Classifier: Send + Sync {+pub trait Classifier<S = ()>: Send + Sync {/// Stable tier represented by `selected_model`, when this classifier defines one.fn routing_tier(&self, _selected_model: &str) -> Option<&'static str> {None@@ -85,7 +85,7 @@ pub trait Classifier: Send + Sync {/// only read it.async fn score(&self,- state: &mut State,+ state: &mut S,request: &mut Request,driver: Option<&Driver>,) -> Result<Classification>;@@ -94,7 +94,6 @@ pub trait Classifier: Send + Sync {#[cfg(test)]mod tests {use super::*;- use crate::StateValue;use switchyard_protocol::text_request;/// Terse `Score` builder for the assertions below.@@ -178,15 +177,14 @@ mod tests {struct RecordingClassifier;#[async_trait]- impl Classifier for RecordingClassifier {+ impl Classifier<bool> for RecordingClassifier {async fn score(&self,- state: &mut State,+ state: &mut bool,request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {- // Stash a marker in `extra` to prove state is threaded mutably.- state.extra.insert("ran".to_string(), StateValue::Count(1));+ *state = true;let target = request.requested_model().unwrap_or("auto").to_string();Ok(Classification::Scores(vec![Score {target,@@ -197,7 +195,7 @@ mod tests {#[tokio::test]async fn classifier_reads_request_and_mutates_state() -> Result<()> {- let mut state = State::default();+ let mut state = false;let mut request = Request {llm_request: text_request(Some("strong".to_string()), "hi"),raw_request: None,@@ -211,7 +209,7 @@ mod tests {classification.argmax(false)?.map(|s| s.target),Some("strong".to_string()));- assert!(matches!(state.extra.get("ran"), Some(StateValue::Count(1))));+ assert!(state);Ok(())}@@ -222,7 +220,7 @@ mod tests {impl Classifier for RewritingClassifier {async fn score(&self,- _state: &mut State,+ _state: &mut (),request: &mut Request,_driver: Option<&Driver>,) -> Result<Classification> {@@ -236,7 +234,7 @@ mod tests {#[tokio::test]async fn classifier_rewrites_the_request_in_place() -> Result<()> {- let mut state = State::default();+ let mut state = ();let mut request = Request {llm_request: text_request(Some("auto".to_string()), "hi"),raw_request: None,diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rsindex b21cc4c23..d3bb7856c 100644--- a/crates/libsy/src/core/processor.rs+++ b/crates/libsy/src/core/processor.rs@@ -1,11 +1,11 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-use crate::{Result, State};+use crate::Result;use async_trait::async_trait;use switchyard_protocol::{AggLlmResponse, Decision, Request, Signals};-/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate [`State`]+/// An event observed by the algorithm. Events are consumed by [`Processor`] to mutate state.////// The two request-bearing variants borrow the request mutably, so a processor may rewrite/// it in place and pass the rewritten request down the chain (see [`Processor::process`]).@@ -15,63 +15,61 @@ pub enum Event<'a> {Request(&'a mut Request),/// An out-of-band agentic-stack signal (tool results, budget updates, …).Signal(&'a Signals),- /// A routing decision the algorithm just made.- Decision(&'a dyn Decision),+ /// A routing decision paired with the request that produced it.+ Decision {+ /// The request classified by the algorithm.+ request: &'a Request,+ /// The routing decision produced for `request`.+ decision: &'a dyn Decision,+ },/// A request about to be sent to a model.ModelRequest(&'a mut Request),/// A buffered response received back from a model.ModelResponse(&'a AggLlmResponse),}-/// Collects events as the algorithm runs and mutates [`State`]+/// Collects events as the algorithm runs and mutates the composition's state.#[async_trait]-pub trait Processor: Send + Sync {+pub trait Processor<S = ()>: Send + Sync {/// Process an event, accumulating facts into `state`.////// A request-bearing event ([`Event::Request`], [`Event::ModelRequest`]) may also be/// rewritten in place; the edit propagates to the rest of the chain and to the model/// call. Most processors only read it.- async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()>;+ async fn process(&self, state: &mut S, event: Event<'_>) -> Result<()>;}#[cfg(test)]mod tests {use super::*;- use crate::StateValue;+ use std::collections::HashMap;use switchyard_protocol::{text_request, text_response};- /// The `State::extra` key each event variant tallies under.+ type TestState = HashMap<&'static str, u32>;++ /// The key each event variant tallies under.fn event_key(event: &Event<'_>) -> &'static str {match event {Event::Request(_) => "requests",Event::Signal(_) => "signals",- Event::Decision(_) => "decisions",+ Event::Decision { .. } => "decisions",Event::ModelRequest(_) => "model_requests",Event::ModelResponse(_) => "model_responses",}}- /// Reads a `StateValue::Count` from `extra`, treating a missing key as zero.- fn count(state: &State, key: &str) -> u32 {- match state.extra.get(key) {- Some(StateValue::Count(n)) => *n,- _ => 0,- }+ /// Reads a count, treating a missing key as zero.+ fn count(state: &TestState, key: &'static str) -> u32 {+ state.get(key).copied().unwrap_or_default()}- /// Tallies each event variant under its own key in [`State::extra`].+ /// Tallies each event variant under its own key.struct CountingProcessor;#[async_trait]- impl Processor for CountingProcessor {- async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {- let entry = state- .extra- .entry(event_key(&event).to_string())- .or_insert(StateValue::Count(0));- if let StateValue::Count(n) = entry {- *n += 1;- }+ impl Processor<TestState> for CountingProcessor {+ async fn process(&self, state: &mut TestState, event: Event<'_>) -> Result<()> {+ *state.entry(event_key(&event)).or_default() += 1;Ok(())}}@@ -102,7 +100,7 @@ mod tests {#[tokio::test]async fn processor_tallies_each_event_variant_into_state() -> Result<()> {let processor = CountingProcessor;- let mut state = State::default();+ let mut state = TestState::default();let mut req = request();let response = text_response(None, "ok");let decision = TestDecision;@@ -119,7 +117,13 @@ mod tests {.process(&mut state, Event::ModelResponse(&response)).await?;processor- .process(&mut state, Event::Decision(&decision))+ .process(+ &mut state,+ Event::Decision {+ request: &req,+ decision: &decision,+ },+ ).await?;processor.process(&mut state, Event::Signal(&signals))@@ -136,7 +140,7 @@ mod tests {#[tokio::test]async fn process_accumulates_state_across_repeated_events() -> Result<()> {let processor = CountingProcessor;- let mut state = State::default();+ let mut state = TestState::default();let mut req = request();for _ in 0..3 {@@ -154,7 +158,7 @@ mod tests {#[async_trait]impl Processor for RewritingProcessor {- async fn process(&self, _state: &mut State, event: Event<'_>) -> Result<()> {+ async fn process(&self, _state: &mut (), event: Event<'_>) -> Result<()> {if let Event::Request(request) | Event::ModelRequest(request) = event {request.llm_request.model = Some("rewritten".to_string());}@@ -164,7 +168,7 @@ mod tests {#[tokio::test]async fn processor_rewrites_the_request_in_place() -> Result<()> {- let mut state = State::default();+ let mut state = ();let mut req = request();assert_eq!(req.requested_model(), Some("auto"));diff --git a/crates/libsy/src/core/state.rs b/crates/libsy/src/core/state.rsindex 58382c76d..b6be3e4ca 100644--- a/crates/libsy/src/core/state.rs+++ b/crates/libsy/src/core/state.rs@@ -1,10 +1,9 @@// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.// SPDX-License-Identifier: Apache-2.0-use std::collections::HashMap;-use std::sync::Arc;+//! Built-in state shared by processors and classifiers across a session.-use tokio::sync::Mutex;+use std::collections::HashMap;use crate::ToolSignals;@@ -17,7 +16,7 @@ pub enum StateValue {Scalar(f32),}-/// State maintained by [`Algorithm`](crate::Algorithm)s+/// Routing facts accumulated across one session's algorithm runs.#[derive(Debug, Clone, Default)]pub struct State {pub turn_count: u32,@@ -27,6 +26,3 @@ pub struct State {pub tool_signals: Option<ToolSignals>,pub extra: HashMap<String, StateValue>,}--/// State shared accross tunrs / sessions-pub type SharedState = Arc<Mutex<State>>;diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex a2eeae382..09f61a793 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -53,7 +53,7 @@ static TOTAL_GAUGES: OnceLock<(ObservableGauge<u64>, ObservableGauge<u64>)> = Onpub(crate) const ALGORITHM_KEY: &str = "algorithm";/// The algorithm label carried by a request context; empty until stamped.-pub(crate) fn algorithm_label<S>(ctx: &Context<S>) -> &str {+pub(crate) fn algorithm_label(ctx: &Context) -> &str {ctx.values.get(ALGORITHM_KEY).map(String::as_str)@@ -135,8 +135,8 @@ pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {/// Runs one algorithm task to completion, recording the run counter, duration/// histogram, span outcome, and failure log when it resolves. Executes inside/// the `libsy.run` span its caller instruments the task with.-pub(crate) async fn observe_run<S>(- ctx: Context<S>,+pub(crate) async fn observe_run(+ ctx: Context,run: impl Future<Output = Result<Response>>,) -> Result<Response> {let started = Instant::now();diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rsindex 63f2cd6a5..70dee0411 100644--- a/crates/libsy/tests/observability.rs+++ b/crates/libsy/tests/observability.rs@@ -29,7 +29,7 @@ use tracing_subscriber::Layer;use switchyard_libsy::algorithms::{FallThrough, LlmTaskClassifier};use switchyard_libsy::{AggLlmResponse, Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmTarget,- LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, SharedState, Step, Usage,+ LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, State, Step, Usage,};use switchyard_protocol::{text_request, text_response, LlmClientError};@@ -725,13 +725,15 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_liblet targets = LlmTargetSet::new(vec![target("weak"), target("strong")]);let weak = targets.get_target("weak")?;let strong = targets.get_target("strong")?;- let router = Arc::new(FallThrough::new(targets).with_classifier(Arc::new(- LlmTaskClassifier::new(target("classifier"), weak, strong, 0.5)?,- )));+ let router = Arc::new(+ FallThrough::<State>::new_with_state(targets).with_classifier(Arc::new(+ LlmTaskClassifier::new(target("classifier"), weak, strong, 0.5)?,+ )),+ );let (trace, _response) = router.run(- Context::<SharedState>::default(),+ Context::default(),Request {llm_request: text_request(Some("auto".to_string()), "classify this"),raw_request: None,diff --git a/crates/protocol/src/envelope.rs b/crates/protocol/src/envelope.rsindex 4560ef7f3..570f3873f 100644--- a/crates/protocol/src/envelope.rs+++ b/crates/protocol/src/envelope.rs@@ -7,24 +7,11 @@use crate::{LlmRequest, LlmResponse, Metadata};use std::collections::HashMap;-/// [`Context`] for an algorithm or llm client.-/// This struct is fed through the entire process and provides unified information to each step+/// Cross-cutting request context passed through algorithms and LLM clients.#[derive(Clone, Debug, Default, PartialEq, Eq)]-pub struct Context<S = ()> {- /// caller specific values+pub struct Context {+ /// Caller-specific values propagated through the request.pub values: HashMap<String, String>,- /// Caller-defined per-session state; `()` when a consumer carries no session state.- pub state: S,-}--impl<S> Context<S> {- /// Create a new [`Context`] with the given state and an empty values map.- pub fn without_state(&self) -> Context<()> {- Context {- values: self.values.clone(),- state: (),- }- }}/// Agentic-stack events fed to an algorithm out of band (e.g. tool results, budgetdiff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rsindex 0c6ac9559..3315a8914 100644--- a/crates/switchyard-server/src/config.rs+++ b/crates/switchyard-server/src/config.rs@@ -9,7 +9,7 @@ use std::path::Path;use std::sync::Arc;use libsy::algorithms::{FallThrough, LlmTaskClassifier, Noop, Passthrough, Random};-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State};use serde::Deserialize;use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};@@ -240,7 +240,7 @@ fn build_algorithm(route_name: &str,config: &RouteConfig,targets: &BTreeMap<String, LlmTarget>,-) -> ServerResult<Arc<dyn Algorithm<SharedState>>> {+) -> ServerResult<Arc<dyn Algorithm>> {match config {RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})),RouteConfig::Random {@@ -276,7 +276,8 @@ fn build_algorithm(LlmTaskClassifier::new(classifier, weak, strong, *threshold).map_err(|error| {ServerError::new(format!("llm_classifier route {route_name}: {error}"))})?;- let router = FallThrough::new(target_set).with_classifier(Arc::new(judge));+ let router =+ FallThrough::<State>::new_with_state(target_set).with_classifier(Arc::new(judge));Ok(Arc::new(router))}}diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex f5d7e7d36..bb0ae635a 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -23,9 +23,7 @@ use axum::response::{IntoResponse, Response};use axum::routing::{get, post};use axum::{Json, Router};use axum_server::tls_rustls::RustlsConfig;-use libsy::{- Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request, SharedState,-};+use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};use serde_json::{json, Value};use tokio::net::{TcpListener, TcpSocket};use tracing::Level;@@ -70,16 +68,14 @@ pub type ServerResult<T> = std::result::Result<T, ServerError>;/// Shared server state used by all endpoint handlers.#[derive(Clone)]pub struct ServerState {- /// For now making all the routes to be of type Algorithm<SharedState>- /// Nachi's PR will help later- routes: Arc<BTreeMap<String, Arc<dyn Algorithm<SharedState>>>>,+ routes: Arc<BTreeMap<String, Arc<dyn Algorithm>>>,metrics: prometheus::Registry,}impl ServerState {/// Creates server state from route model IDs and their libsy algorithms.pub fn new(- routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm<SharedState>>)>,+ routes: impl IntoIterator<Item = (String, Arc<dyn Algorithm>)>,) -> ServerResult<Self> {let mut entries = BTreeMap::new();for (model, algorithm) in routes {@@ -106,7 +102,7 @@ impl ServerState {self.routes.keys().map(String::as_str)}- fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm<SharedState>>> {+ fn algorithm_for_model(&self, model: &str) -> Option<Arc<dyn Algorithm>> {self.routes.get(model).map(Arc::clone)}}@@ -361,7 +357,7 @@ fn resolve_route(metadata: Metadata,body: Value,wire_format: WireFormat,-) -> std::result::Result<(Arc<dyn Algorithm<SharedState>>, Request, String), Response> {+) -> std::result::Result<(Arc<dyn Algorithm>, Request, String), Response> {let llm_request = decode_request(wire_format, &body).map_err(|error| invalid_body_error(error.to_string()))?;let requested_model = llm_request@@ -403,10 +399,7 @@ async fn handle_llm_request(Ok(resolved) => resolved,Err(response) => return response,};- let (trace, response) = match algorithm- .run(Context::<SharedState>::default(), request)- .await- {+ let (trace, response) = match algorithm.run(Context::default(), request).await {Ok(result) => result,Err(error) => return algorithm_error(error),};diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex c8bb644d1..e75fe5445 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -19,7 +19,7 @@ use axum::{Json, Router};use http_body_util::BodyExt;use libsy::algorithms::{FallThrough, Random};use libsy::stage_router::{PickerMode, StageClassifier};-use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, SharedState};+use libsy::{Algorithm, LlmTarget, LlmTargetSet, RoutedLlmClient, State as AlgorithmState};use serde_json::{json, Value};use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient};use switchyard_server::config::load_server_state;@@ -157,8 +157,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server}).collect(),);- let algorithm: Arc<dyn Algorithm<SharedState>> =- Arc::new(Random::new(target_set, None, None)?);+ let algorithm: Arc<dyn Algorithm> = Arc::new(Random::new(target_set, None, None)?);Ok(((*route_model).to_string(), algorithm))}).collect::<TestResult<Vec<_>>>()?;@@ -462,8 +461,9 @@ fn stage_router_state(upstream: &MockUpstream, mode: PickerMode) -> TestResult<Sllm_client: Some(weak),},]);- let stage: Arc<dyn Algorithm<SharedState>> = Arc::new(- FallThrough::new(targets).with_classifier(Arc::new(StageClassifier::new(mode, 0.5))),+ let stage: Arc<dyn Algorithm> = Arc::new(+ FallThrough::<AlgorithmState>::new_with_state(targets)+ .with_classifier(Arc::new(StageClassifier::new(mode, 0.5))),);Ok(ServerState::new([("switchyard/stage".to_string(), stage)])?)}