Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): affinity - sub-agent aware routing (#125)

Greg Clark5a142c8ab39f
diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs
index c2b500704..b81e62217 100644
--- a/crates/libsy/src/algorithms.rs
+++ b/crates/libsy/src/algorithms.rs
@@ -17,3 +17,5 @@ pub use llm_class::{ClassifierDecision, ClassifierTier, LlmClassifier};
pub use noop::{Noop, NoopDecision};
pub use rand::{Random, RandomDecision};
pub use subagent_override::{SubagentDecision, SubagentOverride};
+
+pub(crate) mod util;
diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs
new file mode 100644
index 000000000..99c18a44f
--- /dev/null
+++ b/crates/libsy/src/algorithms/util.rs
@@ -0,0 +1,7 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+mod affinity;
+
+#[allow(unused_imports)]
+pub(crate) use affinity::AffinityRouter;
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
new file mode 100644
index 000000000..70e99f804
--- /dev/null
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -0,0 +1,567 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Model affinity as a single SDK component.
+//!
+//! [`AffinityRouter`] retains the first model chosen for a request's stable identity and
+//! forces that model on later requests sharing the identity. It is one object that plays
+//! both SDK roles, so registering it as a processor and a classifier cannot drift apart:
+//!
+//! - As a [`Processor`] it *writes* the assignment: it captures the request's identity on
+//! [`Event::Request`] and binds it to the chosen model on [`Event::Decision`], the first
+//! assignment for an identity winning.
+//! - As a [`Classifier`] it *reads* the assignment: it scores the retained model with
+//! confidence `1.0`, or returns no scores to abstain.
+//!
+//! Identity is derived from correlation metadata: by default, a request is keyed by its
+//! session, and a sub-agent is keyed more finely by `session + agent` — a subset of its
+//! session. [`AffinityRouter::for_subagents`] narrows affinity to explicitly identified
+//! child agents, leaving root traffic to later classifiers on every turn.
+
+#![allow(dead_code)]
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Mutex;
+
+use async_trait::async_trait;
+use switchyard_protocol::Request;
+
+use crate::{Classification, Classifier, Event, Processor, Score, State};
+
+/// Upper bound on retained assignments, keeping the process-local map from growing
+/// without limit; the oldest entry is evicted once the bound is reached.
+const MAX_ASSIGNMENTS: usize = 4096;
+
+/// The stable identity a model assignment is retained against.
+///
+/// A sub-agent request is keyed by `session + agent` and a root request by session alone,
+/// so a sub-agent's assignment is scoped within — but distinct from — its session's.
+#[derive(Clone, Hash, PartialEq, Eq)]
+enum AffinityKey {
+ /// One model per session, for root-agent traffic.
+ Session(String),
+ /// One model per identified child agent within a session.
+ Subagent { session: String, agent: String },
+}
+
+/// The affinity memory: the retained model per identity, plus the key captured from the
+/// current turn's request while its decision is pending.
+#[derive(Default)]
+struct AffinityAssignments {
+ /// Identity → retained model, first assignment winning.
+ assignments: HashMap<AffinityKey, String>,
+ /// Key captured on this turn's [`Event::Request`], consumed by its [`Event::Decision`].
+ pending: Option<AffinityKey>,
+}
+
+/// Retains a model per request identity and forces it on later matching requests.
+///
+/// Register the same instance as both a processor and a classifier; the two roles share
+/// the retained assignments through this instance's interior storage. Because a decision
+/// event carries only the chosen model, the request's identity is captured first (on
+/// [`Event::Request`]) and consumed when the decision arrives — this assumes a turn's
+/// request is observed before its decision, as the driving algorithm folds a turn in order.
+///
+/// [`with_latch_only`](Self::with_latch_only) narrows *which* models are retained — a
+/// decision for any other model routes normally but is not latched (the escalation latch:
+/// retain only the strong tier, never the weak one).
+#[derive(Default)]
+pub struct AffinityRouter {
+ /// When set, only these models are retained; a decision for any other model is not latched.
+ latch_only: Option<HashSet<String>>,
+ /// Whether root-session requests should abstain instead of being retained.
+ subagents_only: bool,
+ /// Retained assignments, shared across this router's processor and classifier roles.
+ ///
+ /// Held on the instance rather than in [`State`] so the two roles share one
+ /// process-local map through a single registered [`Arc`](std::sync::Arc); bounded by
+ /// [`MAX_ASSIGNMENTS`].
+ assignments: Mutex<AffinityAssignments>,
+}
+
+impl AffinityRouter {
+ /// Creates a router that latches every decision.
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Creates a router that retains assignments only for explicitly identified child agents.
+ ///
+ /// Root-agent requests always abstain, so a later classifier selects them on every turn.
+ pub fn for_subagents() -> Self {
+ Self {
+ subagents_only: true,
+ ..Self::default()
+ }
+ }
+
+ /// Restricts latching to `models`; a decision for any other model routes but is not
+ /// retained.
+ pub fn with_latch_only(mut self, models: impl IntoIterator<Item = impl Into<String>>) -> Self {
+ self.latch_only = Some(models.into_iter().map(Into::into).collect());
+ self
+ }
+
+ /// Whether a decision for `model` should be retained.
+ fn should_latch(&self, model: &str) -> bool {
+ self.latch_only
+ .as_ref()
+ .is_none_or(|set| set.contains(model))
+ }
+
+ /// Derives the stable identity this router should retain for `request`.
+ fn affinity_key(&self, request: &Request) -> Option<AffinityKey> {
+ let metadata = request.metadata.as_ref()?;
+ let session = metadata.session_id.clone()?;
+ if metadata.is_subagent {
+ Some(AffinityKey::Subagent {
+ session,
+ agent: metadata.agent_id.clone()?,
+ })
+ } else if self.subagents_only {
+ None
+ } else {
+ Some(AffinityKey::Session(session))
+ }
+ }
+}
+
+#[async_trait]
+impl Processor for AffinityRouter {
+ async fn process(&self, _state: &mut State, event: Event<'_>) -> crate::Result<()> {
+ // No `.await` is held across this guard, so a std Mutex is sufficient; recover from
+ // poisoning rather than panicking, since a stale assignment map is harmless.
+ let mut fact = self
+ .assignments
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner());
+ match event {
+ // Capture the identity now; the model it maps to is only known at the decision.
+ Event::Request(request) => {
+ fact.pending = self.affinity_key(request);
+ }
+ // Bind the captured identity to the chosen model, keeping any earlier winner.
+ Event::Decision(decision) => {
+ let model = decision.selected_model();
+ let Some(key) = fact.pending.take() else {
+ return Ok(());
+ };
+ // Latch only permitted models; others consume `pending` but are not retained.
+ if self.should_latch(model) && !fact.assignments.contains_key(&key) {
+ evict_if_full(&mut fact.assignments);
+ fact.assignments.insert(key, model.to_string());
+ }
+ }
+ _ => {}
+ }
+ Ok(())
+ }
+}
+
+#[async_trait]
+impl Classifier for AffinityRouter {
+ async fn score(
+ &self,
+ _state: &mut State,
+ request: &Request,
+ _driver: Option<&crate::Driver>,
+ ) -> crate::Result<Classification> {
+ let Some(key) = self.affinity_key(request) else {
+ return Ok(Classification::Scores(Vec::new()));
+ };
+ let assigned = self
+ .assignments
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .assignments
+ .get(&key)
+ .cloned();
+ Ok(Classification::Scores(match assigned {
+ Some(target) => vec![Score {
+ confidence: 1.0,
+ target,
+ }],
+ None => Vec::new(),
+ }))
+ }
+}
+
+/// Evicts one arbitrary assignment when the map has reached [`MAX_ASSIGNMENTS`].
+fn evict_if_full(assignments: &mut HashMap<AffinityKey, String>) {
+ if assignments.len() >= MAX_ASSIGNMENTS {
+ if let Some(evicted) = assignments.keys().next().cloned() {
+ assignments.remove(&evicted);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use std::sync::Arc;
+
+ use switchyard_protocol::{text_request, Decision, Metadata, Signals};
+
+ /// Boxed, thread-safe error type keeping the test helpers ergonomic; libsy errors
+ /// convert into it through `?`.
+ type BoxErr = Box<dyn std::error::Error + Send + Sync>;
+
+ /// A decision that reports a fixed selected model.
+ struct FixedDecision(&'static str);
+
+ impl Decision for FixedDecision {
+ fn selected_model(&self) -> &str {
+ self.0
+ }
+
+ fn reasoning(&self) -> Option<&str> {
+ None
+ }
+
+ fn as_any(&self) -> &dyn std::any::Any {
+ self
+ }
+ }
+
+ fn request(metadata: Metadata) -> Request {
+ Request {
+ llm_request: text_request(Some("auto".to_string()), "hi"),
+ raw_request: None,
+ metadata: Some(metadata),
+ }
+ }
+
+ fn session(session_id: &str, agent_id: &str) -> Metadata {
+ Metadata {
+ session_id: Some(session_id.to_string()),
+ agent_id: Some(agent_id.to_string()),
+ ..Metadata::default()
+ }
+ }
+
+ fn subagent(agent_id: &str, task_id: &str) -> Metadata {
+ Metadata {
+ session_id: Some("session-1".to_string()),
+ agent_id: Some(agent_id.to_string()),
+ task_id: Some(task_id.to_string()),
+ is_subagent: true,
+ ..Metadata::default()
+ }
+ }
+
+ /// Folds a request and its decision through the router, retaining `model`.
+ async fn retain(
+ router: &AffinityRouter,
+ state: &mut State,
+ request: &Request,
+ model: &'static str,
+ ) -> Result<(), BoxErr> {
+ router.process(state, Event::Request(request)).await?;
+ router
+ .process(state, Event::Decision(&FixedDecision(model)))
+ .await?;
+ Ok(())
+ }
+
+ /// Scores through the definitive classification variant used by affinity.
+ async fn scores(
+ classifier: &dyn Classifier,
+ state: &mut State,
+ request: &Request,
+ ) -> Result<Vec<Score>, BoxErr> {
+ match classifier.score(state, request, None).await? {
+ Classification::Scores(scores) => Ok(scores),
+ Classification::Ambiguous(_) => Err("affinity never returns ambiguous scores".into()),
+ }
+ }
+
+ #[tokio::test]
+ async fn session_retains_first_model_across_requests() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ let first = request(session("session-1", "agent-a"));
+ retain(&router, &mut state, &first, "model-a").await?;
+
+ // A different agent in the same session is scored onto the retained model.
+ let second = request(session("session-1", "agent-b"));
+ let scores = scores(&router, &mut state, &second).await?;
+ assert_eq!(scores.len(), 1);
+ assert_eq!(scores[0].confidence, 1.0);
+ assert_eq!(scores[0].target, "model-a");
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn subagent_only_retains_children_without_latching_root_traffic() -> Result<(), BoxErr> {
+ let router = AffinityRouter::for_subagents();
+ let mut state = State::default();
+
+ let root = request(session("session-1", "root-agent"));
+ retain(&router, &mut state, &root, "model-a").await?;
+ assert!(scores(&router, &mut state, &root).await?.is_empty());
+
+ let first_child_turn = request(subagent("child-1", "task-1"));
+ retain(&router, &mut state, &first_child_turn, "model-b").await?;
+ let later_child_turn = request(subagent("child-1", "task-2"));
+ let scores = scores(&router, &mut state, &later_child_turn).await?;
+ assert_eq!(
+ scores.first().map(|score| score.target.as_str()),
+ Some("model-b")
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn first_decision_wins() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ let req = request(session("session-1", "agent-a"));
+ retain(&router, &mut state, &req, "model-a").await?;
+ // A later decision for the same identity must not overwrite the first.
+ retain(&router, &mut state, &req, "model-b").await?;
+
+ let scores = scores(&router, &mut state, &req).await?;
+ assert_eq!(
+ scores.first().map(|score| score.target.as_str()),
+ Some("model-a")
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn subagent_is_keyed_by_agent_not_task() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ let first = request(subagent("child-1", "task-1"));
+ retain(&router, &mut state, &first, "model-a").await?;
+
+ // Same child, different task: still scored onto the retained model.
+ let second = request(subagent("child-1", "task-2"));
+ let scores = scores(&router, &mut state, &second).await?;
+ assert_eq!(
+ scores.first().map(|score| score.target.as_str()),
+ Some("model-a")
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn distinct_subagents_are_assigned_independently() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ // One child in the session is pinned...
+ retain(
+ &router,
+ &mut state,
+ &request(subagent("child-1", "task-1")),
+ "model-a",
+ )
+ .await?;
+
+ // ...a sibling child in the same session has no assignment of its own yet.
+ let sibling = request(subagent("child-2", "task-1"));
+ assert!(scores(&router, &mut state, &sibling).await?.is_empty());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn subagent_does_not_inherit_session_assignment() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ // The session root is pinned, but a sub-agent is keyed separately...
+ retain(
+ &router,
+ &mut state,
+ &request(session("session-1", "root-1")),
+ "model-a",
+ )
+ .await?;
+
+ // ...so the sub-agent abstains until it is assigned in its own right.
+ let child = request(subagent("child-1", "task-1"));
+ assert!(scores(&router, &mut state, &child).await?.is_empty());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn classifier_abstains_without_a_session() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ // No session id at all: nothing to key on.
+ let req = request(Metadata::default());
+ assert!(scores(&router, &mut state, &req).await?.is_empty());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn one_router_serves_both_roles() -> Result<(), BoxErr> {
+ // The same instance is registered under both SDK roles; a decision folded in via
+ // the processor handle is read back via the classifier handle.
+ let router = Arc::new(AffinityRouter::new());
+ let processor: Arc<dyn Processor> = router.clone();
+ let classifier: Arc<dyn Classifier> = router;
+ let mut state = State::default();
+
+ let first = request(session("session-1", "agent-a"));
+ processor
+ .process(&mut state, Event::Request(&first))
+ .await?;
+ processor
+ .process(&mut state, Event::Decision(&FixedDecision("model-a")))
+ .await?;
+
+ let second = request(session("session-1", "agent-b"));
+ let scores = scores(classifier.as_ref(), &mut state, &second).await?;
+ assert_eq!(
+ scores.first().map(|score| score.target.as_str()),
+ Some("model-a")
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn decision_without_a_request_is_ignored() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ // A decision with no captured identity (no prior request) stores nothing.
+ router
+ .process(&mut state, Event::Decision(&FixedDecision("model-a")))
+ .await?;
+
+ let req = request(session("session-1", "agent-a"));
+ assert!(scores(&router, &mut state, &req).await?.is_empty());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn unrelated_events_preserve_the_pending_identity() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ let req = request(session("session-1", "agent-a"));
+ router.process(&mut state, Event::Request(&req)).await?;
+ // A stray signal between the request and its decision must not drop the captured
+ // identity, nor create an assignment of its own.
+ router
+ .process(&mut state, Event::Signal(&Signals {}))
+ .await?;
+ router
+ .process(&mut state, Event::Decision(&FixedDecision("model-a")))
+ .await?;
+
+ let scores = scores(&router, &mut state, &req).await?;
+ assert_eq!(
+ scores.first().map(|score| score.target.as_str()),
+ Some("model-a")
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn distinct_sessions_are_assigned_independently() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ retain(
+ &router,
+ &mut state,
+ &request(session("session-1", "agent-a")),
+ "model-a",
+ )
+ .await?;
+ retain(
+ &router,
+ &mut state,
+ &request(session("session-2", "agent-a")),
+ "model-b",
+ )
+ .await?;
+
+ let first = scores(&router, &mut state, &request(session("session-1", "other"))).await?;
+ let second = scores(&router, &mut state, &request(session("session-2", "other"))).await?;
+ assert_eq!(
+ first.first().map(|score| score.target.as_str()),
+ Some("model-a")
+ );
+ assert_eq!(
+ second.first().map(|score| score.target.as_str()),
+ Some("model-b")
+ );
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn subagent_without_an_agent_id_is_not_keyed() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ // The sub-agent flag is set but no agent id is present, so no key can be formed;
+ // the request is neither retained nor scored.
+ let metadata = Metadata {
+ session_id: Some("session-1".to_string()),
+ is_subagent: true,
+ ..Metadata::default()
+ };
+ let req = request(metadata);
+ retain(&router, &mut state, &req, "model-a").await?;
+ assert!(scores(&router, &mut state, &req).await?.is_empty());
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn assignments_are_bounded_by_the_cap() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new();
+ let mut state = State::default();
+
+ // One distinct session past the cap forces exactly one eviction.
+ for index in 0..=MAX_ASSIGNMENTS {
+ let session_id = format!("session-{index}");
+ retain(
+ &router,
+ &mut state,
+ &request(session(&session_id, "agent-a")),
+ "model-a",
+ )
+ .await?;
+ }
+
+ let len = router
+ .assignments
+ .lock()
+ .unwrap_or_else(|poisoned| poisoned.into_inner())
+ .assignments
+ .len();
+ assert_eq!(len, MAX_ASSIGNMENTS);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn latch_only_retains_matching_models() -> Result<(), BoxErr> {
+ let router = AffinityRouter::new().with_latch_only(["strong"]);
+ let mut state = State::default();
+ let req = request(session("session-1", "agent-a"));
+
+ // A "weak" decision is not retained — a later turn is not latched.
+ retain(&router, &mut state, &req, "weak").await?;
+ assert!(scores(&router, &mut state, &req).await?.is_empty());
+
+ // A "strong" decision is retained — later turns latch onto it.
+ retain(&router, &mut state, &req, "strong").await?;
+ assert_eq!(
+ scores(&router, &mut state, &req)
+ .await?
+ .first()
+ .map(|s| s.target.as_str()),
+ Some("strong")
+ );
+ Ok(())
+ }
+}
diff --git a/crates/libsy/src/core/driver.rs b/crates/libsy/src/core/driver.rs
index 3ad94de2b..33beb48bd 100644
--- a/crates/libsy/src/core/driver.rs
+++ b/crates/libsy/src/core/driver.rs
@@ -63,7 +63,7 @@ type BoxAny = Box<dyn Any + Send>;
type StepResult = Result<DriverStep>;
// TODO make request timeout configurable
-const FULFILL_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
+const FULFILL_REQUEST_TIMEOUT: Duration = Duration::from_mins(10);
/// One item on the stream returned by [`TypeErasedDriver::stream`].
pub enum DriverStep {