Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: Session affinity stores Category for fallback (#692)

Graham Kingc89c2158a1f2
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index 0d04f297e..698f0a9dd 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -3,8 +3,8 @@
//! Model affinity as a single SDK component.
//!
-//! [`AffinityRouter`] retains the first model chosen for a request's stable identity and
-//! forces that model on later requests sharing the identity. It is one object that plays
+//! [`AffinityRouter`] retains the first model and category chosen for a request's stable identity
+//! and forces that decision on later requests sharing the identity. It is one object that plays
//! both SDK roles, so registering it as a processor and a classifier cannot drift apart:
//!
//! - As a [`Processor`] it *writes* the assignment: [`Event::Decision`] carries the request
@@ -85,7 +85,7 @@ pub struct AffinityRouter {
///
/// Held on the instance so the two roles share one process-local map through a
/// single registered [`Arc`](std::sync::Arc); bounded by [`MAX_ASSIGNMENTS`].
- assignments: Mutex<HashMap<RoutingIdentity, ModelId>>,
+ assignments: Mutex<HashMap<RoutingIdentity, (ModelId, Option<Category>)>>,
/// Whether the "no identity to key on" warning has already been emitted.
unkeyed_warning_emitted: AtomicBool,
}
@@ -177,6 +177,7 @@ where
if let Event::Decision {
request,
selected_model_id,
+ category,
..
} = event
&& let Some(key) = self.affinity_key(request)
@@ -185,7 +186,7 @@ where
let writable = self.release_on_user_turn || !assignments.contains_key(&key);
if self.should_latch(selected_model_id) && writable {
evict_if_full(&mut assignments);
- assignments.insert(key, selected_model_id.clone());
+ assignments.insert(key, (selected_model_id.clone(), category));
}
}
Ok(())
@@ -249,7 +250,7 @@ where
let mut assignments = self.assignments.lock();
let assigned = assignments.get(&key).cloned();
let assigned = match assigned.as_ref() {
- Some(target) if !available.is_empty() && !available.contains(target) => {
+ Some((target, _)) if !available.is_empty() && !available.contains(target) => {
assignments.remove(&key);
None
}
@@ -260,10 +261,10 @@ where
}
Ok((
Classification::Scores(match assigned {
- Some(target) => vec![Score {
+ Some((target, category)) => vec![Score {
confidence: 1.0,
target: target.clone(),
- category: None,
+ category: category.clone(),
}],
None => Vec::new(),
}),
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index 6af9bcbe4..ceacc8b3c 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -14,8 +14,8 @@ pub struct Score {
/// The target (model / tier) being recommended.
pub target: ModelId,
/// The category `target` was drawn from, when the classifier picked one. The rest of
- /// that category is what the turn falls through on failure, so a decision made without
- /// a category — an affinity replay, say — leaves this `None`.
+ /// that category is what the turn falls through on failure. A decision made without
+ /// a category leaves this `None`.
pub category: Option<Category>,
}
diff --git a/crates/libsy/src/core/processor.rs b/crates/libsy/src/core/processor.rs
index 538f11157..a94c047d3 100644
--- a/crates/libsy/src/core/processor.rs
+++ b/crates/libsy/src/core/processor.rs
@@ -29,8 +29,7 @@ pub enum Event<'a> {
/// The model selected for `request`.
selected_model_id: &'a ModelId,
/// The category `selected_model_id` was drawn from, when the deciding
- /// classifier picked one. `None` for a decision made without a category,
- /// such as an affinity replay.
+ /// classifier picked one. `None` for a decision made without a category.
category: Option<Category>,
/// Offered so a processor can inspect the runtime model categories.
driver: &'a Driver,