Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

refactor(libsy): classifier::score returns the response alongside the score (#203)

Lin Jia6c3f2651bbd7
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 412a950d5..0d35142bf 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -90,12 +90,15 @@ impl<S: Send> Classifier<S> for DefaultTarget {
_state: &mut S,
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
// Zero confidence: this is a fallback, not a judgement.
- Ok(Classification::Scores(vec![Score {
- target: self.target.clone(),
- confidence: 0.0,
- }]))
+ Ok((
+ Classification::Scores(vec![Score {
+ target: self.target.clone(),
+ confidence: 0.0,
+ }]),
+ None,
+ ))
}
}
@@ -204,7 +207,7 @@ where
ctx.exclude_target(target);
}
let session_state = self.session_state(&request);
- let (target, decision) = match session_state {
+ let (target, decision, served) = match session_state {
Some(state) => {
let mut state = state.lock().await;
self.route(&mut state, &ctx, &driver, &mut request).await?
@@ -215,15 +218,25 @@ where
}
};
- self.call_with_overflow_fallback(
- ctx,
- &driver,
- target,
- decision,
- request,
- session.as_deref(),
- )
- .await
+ // A classifier that already called a model — because deciding required one, and that
+ // call also answers the turn — hands its response back here, so the turn is not paid
+ // for twice. There is no outbound call left to overflow, so the fallback is skipped.
+ // Nothing reads it on the way out: streamed or buffered, it reaches the caller
+ // untouched.
+ match served {
+ Some(response) => Ok(response),
+ None => {
+ self.call_with_overflow_fallback(
+ ctx,
+ &driver,
+ target,
+ decision,
+ request,
+ session.as_deref(),
+ )
+ .await
+ }
+ }
}
fn eligible_targets(&self, ctx: &Context) -> usize {
@@ -332,7 +345,7 @@ where
ctx: &Context,
driver: &Driver,
request: &mut Request,
- ) -> Result<(crate::LlmTarget, Arc<dyn Decision>)> {
+ ) -> Result<(crate::LlmTarget, Arc<dyn Decision>, Option<Response>)> {
// 1. Processor chain accumulates request-side facts into the composition's state.
for processor in &self.processors {
processor.process(state, Event::Request(request)).await?;
@@ -342,11 +355,15 @@ where
// per-request driver is offered to each — driver-backed classifiers use it.
let mut maybe_score: Option<Score> = None;
let mut deciding: Option<&Arc<dyn Classifier<S>>> = None;
+ let mut served: Option<Response> = None;
for classifier in &self.classifiers {
- let scores = classifier.score(state, request, Some(driver)).await?;
+ let (scores, response) = classifier.score(state, request, Some(driver)).await?;
maybe_score = scores.argmax(false)?;
if maybe_score.is_some() {
deciding = Some(classifier);
+ // Only the deciding classifier's response answers the turn; an abstaining
+ // classifier selected nothing for it to be the answer to.
+ served = response;
break;
}
}
@@ -384,7 +401,7 @@ where
processor.process(state, event).await?;
}
- Ok((target, decision))
+ Ok((target, decision, served))
}
}
@@ -515,15 +532,18 @@ mod tests {
_state: &mut (),
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
- Ok(Classification::Scores(
- self.0
- .iter()
- .map(|s| Score {
- confidence: s.confidence,
- target: s.target.clone(),
- })
- .collect(),
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((
+ Classification::Scores(
+ self.0
+ .iter()
+ .map(|s| Score {
+ confidence: s.confidence,
+ target: s.target.clone(),
+ })
+ .collect(),
+ ),
+ None,
))
}
}
@@ -872,9 +892,9 @@ mod tests {
_state: &mut (),
_request: &mut Request,
driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
match driver {
- Some(_) => Ok(Classification::Scores(vec![score("strong", 1.0)])),
+ Some(_) => Ok((Classification::Scores(vec![score("strong", 1.0)]), None)),
None => Err(test_error("expected a driver")),
}
}
@@ -949,7 +969,7 @@ mod tests {
_state: &mut (),
request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
*self.0.lock() = request
.llm_request
.messages
@@ -960,7 +980,7 @@ mod tests {
.llm_request
.messages
.push(Message::text(Role::User, "classifier"));
- Ok(Classification::Scores(vec![score("strong", 1.0)]))
+ Ok((Classification::Scores(vec![score("strong", 1.0)]), None))
}
}
@@ -1025,9 +1045,9 @@ mod tests {
state: &mut TurnState,
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
let target = if state.count >= 2 { "strong" } else { "weak" };
- Ok(Classification::Scores(vec![score(target, 1.0)]))
+ Ok((Classification::Scores(vec![score(target, 1.0)]), None))
}
}
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 06dac2112..367748f51 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -351,7 +351,7 @@ impl Classifier<State> for TaskClassifier {
state: &mut State,
request: &mut Request,
driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
self.classifier.score(state, request, driver).await
}
}
@@ -367,7 +367,7 @@ impl Classifier<State> for LlmTaskClassifier {
state: &mut State,
request: &mut Request,
driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
self.classifier.score(state, request, driver).await
}
}
diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs
index 60471a125..e7adbd2f2 100644
--- a/crates/libsy/src/algorithms/rand.rs
+++ b/crates/libsy/src/algorithms/rand.rs
@@ -111,11 +111,14 @@ where
_state: &mut S,
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
- Ok(Classification::Scores(vec![Score {
- confidence: 1.0,
- target: self.select_target(),
- }]))
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((
+ Classification::Scores(vec![Score {
+ confidence: 1.0,
+ target: self.select_target(),
+ }]),
+ None,
+ ))
}
}
@@ -378,6 +381,7 @@ mod tests {
let retained = affinity
.score(&mut state, &mut request, None)
.await?
+ .0
.argmax(false)?;
assert_eq!(
retained.map(|score| score.target),
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 8161c1888..da2a69f8f 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -55,13 +55,13 @@ impl Classifier<State> for SourceStamp {
state: &mut State,
request: &mut Request,
driver: Option<&Driver>,
- ) -> Result<Classification> {
- let classification = self.inner.score(state, request, driver).await?;
+ ) -> Result<(Classification, Option<Response>)> {
+ let (classification, served) = self.inner.score(state, request, driver).await?;
// An abstaining classifier passes the turn on, so it is not its to claim.
if matches!(&classification, Classification::Scores(scores) if !scores.is_empty()) {
record_decision_source(state, self.source);
}
- Ok(classification)
+ Ok((classification, served))
}
}
@@ -254,11 +254,14 @@ mod tests {
_state: &mut State,
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
- Ok(Classification::Scores(vec![crate::Score {
- target: self.0.to_string(),
- confidence: 1.0,
- }]))
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((
+ Classification::Scores(vec![crate::Score {
+ target: self.0.to_string(),
+ confidence: 1.0,
+ }]),
+ None,
+ ))
}
}
@@ -272,8 +275,8 @@ mod tests {
_state: &mut State,
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
- Ok(Classification::Ambiguous(vec![]))
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((Classification::Ambiguous(vec![]), None))
}
}
diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
index ba7cd1d65..e9a1d28e5 100644
--- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs
+++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs
@@ -48,11 +48,14 @@ impl Classifier for AlwaysOrchestrator {
_state: &mut (),
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
- Ok(Classification::Scores(vec![Score {
- confidence: 0.5,
- target: "orchestrator".to_string(),
- }]))
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((
+ Classification::Scores(vec![Score {
+ confidence: 0.5,
+ target: "orchestrator".to_string(),
+ }]),
+ None,
+ ))
}
}
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index ee6fdc22d..82283d041 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -180,18 +180,21 @@ where
_state: &mut S,
request: &mut Request,
_driver: Option<&crate::Driver>,
- ) -> crate::Result<Classification> {
+ ) -> crate::Result<(Classification, Option<crate::Response>)> {
let Some(key) = self.affinity_key(request) else {
- return Ok(Classification::Scores(Vec::new()));
+ return Ok((Classification::Scores(Vec::new()), None));
};
let assigned = self.assignments.lock().get(&key).cloned();
- Ok(Classification::Scores(match assigned {
- Some(target) => vec![Score {
- confidence: 1.0,
- target,
- }],
- None => Vec::new(),
- }))
+ Ok((
+ Classification::Scores(match assigned {
+ Some(target) => vec![Score {
+ confidence: 1.0,
+ target,
+ }],
+ None => Vec::new(),
+ }),
+ None,
+ ))
}
}
@@ -309,7 +312,7 @@ mod tests {
state: &mut (),
request: &mut Request,
) -> Result<Vec<Score>, BoxErr> {
- match classifier.score(state, request, None).await? {
+ match classifier.score(state, request, None).await?.0 {
Classification::Scores(scores) => Ok(scores),
Classification::Ambiguous(_) => Err("affinity never returns ambiguous scores".into()),
}
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index b9ea6b98e..4fc594ae7 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -15,8 +15,8 @@ use serde_json::Value;
use switchyard_protocol::{completion_text, AggLlmResponse};
use crate::{
- Classification, Classifier, Context, Decision, Driver, LibsyError, LlmTarget, Request, Result,
- State,
+ Classification, Classifier, Context, Decision, Driver, LibsyError, LlmTarget, Request,
+ Response, Result, State,
};
#[derive(Clone, Debug, Default)]
@@ -128,7 +128,7 @@ where
state: &mut State,
request: &mut Request,
driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
// A missing driver is a broken composition, not an unavailable judge.
let Some(driver) = driver else {
return Err(LibsyError::AlgorithmError {
@@ -139,7 +139,8 @@ where
});
};
let verdict = self.verdict(state, request, driver).await;
- Ok(self.policy.to_classification(verdict.as_ref()))
+ // A judge consultation is a side call, never the turn's answer.
+ Ok((self.policy.to_classification(verdict.as_ref()), None))
}
}
@@ -357,7 +358,8 @@ mod tests {
classifier.score(&mut state, &mut request, Some(&driver)),
serve
);
- selected(classification?)
+ let (classification, _) = classification?;
+ selected(classification)
}
#[tokio::test]
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 86519cb1c..3ed296cb1 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -484,12 +484,12 @@ impl Classifier<State> for StageClassifier {
state: &mut State,
request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<crate::Response>)> {
let tool_signals = &state.tool_signals;
let Some(signal) = tool_signals else {
// No tool activity yet — nothing to score, so the signals have no
// opinion, same as a below-threshold turn.
- return Ok(Self::abstain(state));
+ return Ok((Self::abstain(state), None));
};
let outcome = pick_tier(signal, self.mode, self.confidence_threshold);
@@ -507,12 +507,15 @@ impl Classifier<State> for StageClassifier {
// ambiguous turn is decided further down the cascade.
self.apply_handoff_note(request, tier, source);
let conf = score.abs();
- Ok(Classification::Scores(vec![Score {
- target: target.to_string(),
- confidence: conf,
- }]))
+ Ok((
+ Classification::Scores(vec![Score {
+ target: target.to_string(),
+ confidence: conf,
+ }]),
+ None,
+ ))
}
- PickOutcome::ConsultClassifier { .. } => Ok(Self::abstain(state)),
+ PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(state), None)),
}
}
}
@@ -616,7 +619,7 @@ mod tests {
let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5)
.score(&mut state, &mut Request::default(), None)
.await?;
- assert!(classification.argmax(false)?.is_none());
+ assert!(classification.0.argmax(false)?.is_none());
assert!(matches!(
state.extra.get(DECISION_SOURCE_KEY),
Some(StateValue::String(source)) if source == "ambiguous"
@@ -635,7 +638,7 @@ mod tests {
let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5)
.score(&mut state, &mut Request::default(), None)
.await?;
- match classification {
+ match classification.0 {
Classification::Scores(scores) => {
assert_eq!(scores.len(), 1);
assert_eq!(scores[0].target, "strong");
@@ -664,7 +667,7 @@ mod tests {
let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5)
.score(&mut state, &mut Request::default(), None)
.await?;
- match classification {
+ match classification.0 {
Classification::Scores(scores) => {
assert_eq!(scores.len(), 1);
assert_eq!(scores[0].target, "weak");
@@ -682,7 +685,7 @@ mod tests {
let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5)
.score(&mut state, &mut Request::default(), None)
.await?;
- assert!(classification.argmax(false)?.is_none());
+ assert!(classification.0.argmax(false)?.is_none());
assert!(matches!(
state.extra.get(DECISION_SOURCE_KEY),
Some(StateValue::String(source)) if source == "ambiguous"
@@ -841,7 +844,7 @@ mod tests {
.score(&mut state, &mut request, None)
.await?;
- assert!(matches!(classification, Classification::Ambiguous(_)));
+ assert!(matches!(classification.0, Classification::Ambiguous(_)));
assert_eq!(trailing_text(&request), Some("hi".to_string()));
Ok(())
}
diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs
index 166c8f121..b80543e5c 100644
--- a/crates/libsy/src/algorithms/util/subagent.rs
+++ b/crates/libsy/src/algorithms/util/subagent.rs
@@ -17,7 +17,7 @@
use async_trait::async_trait;
-use crate::{Classification, Classifier, Driver, Metadata, Request, Result, Score};
+use crate::{Classification, Classifier, Driver, Metadata, Request, Response, Result, Score};
/// Scores a fixed worker target for delegated sub-agent work; abstains otherwise.
pub struct SubagentOverride {
@@ -47,21 +47,24 @@ where
_state: &mut S,
request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
// Delegated *work* only. A harness maintenance turn (e.g. Codex `compact`) carries
// sub-agent lineage but is not delegated work, so it abstains and routes normally.
let is_delegated_work = request
.metadata
.as_ref()
.is_some_and(Metadata::is_subagent_work);
- Ok(Classification::Scores(if is_delegated_work {
- vec![Score {
- confidence: 1.0,
- target: self.worker.clone(),
- }]
- } else {
- Vec::new()
- }))
+ Ok((
+ Classification::Scores(if is_delegated_work {
+ vec![Score {
+ confidence: 1.0,
+ target: self.worker.clone(),
+ }]
+ } else {
+ Vec::new()
+ }),
+ None,
+ ))
}
}
@@ -94,7 +97,7 @@ mod tests {
let classification = SubagentOverride::new("worker")
.score(&mut state, &mut request(headers), None)
.await?;
- Ok(classification.argmax(false)?.map(|score| score.target))
+ Ok(classification.0.argmax(false)?.map(|score| score.target))
}
#[tokio::test]
@@ -146,7 +149,7 @@ mod tests {
None,
)
.await?;
- match classification {
+ match classification.0 {
Classification::Scores(scores) => {
assert_eq!(scores.len(), 1);
assert_eq!(scores[0].confidence, 1.0);
diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs
index 2ab90f6a0..863c5dedd 100644
--- a/crates/libsy/src/core/classifier.rs
+++ b/crates/libsy/src/core/classifier.rs
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
-use crate::{Driver, LibsyError, Result};
+use crate::{Driver, LibsyError, Response, Result};
use async_trait::async_trait;
use switchyard_protocol::Request;
@@ -88,7 +88,7 @@ pub trait Classifier<S = ()>: Send + Sync {
state: &mut S,
request: &mut Request,
driver: Option<&Driver>,
- ) -> Result<Classification>;
+ ) -> Result<(Classification, Option<Response>)>;
}
#[cfg(test)]
@@ -183,13 +183,16 @@ mod tests {
state: &mut bool,
request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
*state = true;
let target = request.requested_model().unwrap_or("auto").to_string();
- Ok(Classification::Scores(vec![Score {
- target,
- confidence: 1.0,
- }]))
+ Ok((
+ Classification::Scores(vec![Score {
+ target,
+ confidence: 1.0,
+ }]),
+ None,
+ ))
}
}
@@ -202,7 +205,7 @@ mod tests {
metadata: None,
};
// A `None` driver is valid: the classifier scored without offloading a model call.
- let classification = RecordingClassifier
+ let (classification, _) = RecordingClassifier
.score(&mut state, &mut request, None)
.await?;
assert_eq!(
@@ -223,12 +226,15 @@ mod tests {
_state: &mut (),
request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
+ ) -> Result<(Classification, Option<Response>)> {
request.llm_request.model = Some("rewritten".to_string());
- Ok(Classification::Scores(vec![Score {
- target: "rewritten".to_string(),
- confidence: 1.0,
- }]))
+ Ok((
+ Classification::Scores(vec![Score {
+ target: "rewritten".to_string(),
+ confidence: 1.0,
+ }]),
+ None,
+ ))
}
}
diff --git a/crates/libsy/tests/prompts.rs b/crates/libsy/tests/prompts.rs
index 4d28e705f..31301f072 100644
--- a/crates/libsy/tests/prompts.rs
+++ b/crates/libsy/tests/prompts.rs
@@ -87,8 +87,8 @@ impl Classifier for Abstains {
_state: &mut (),
_request: &mut Request,
_driver: Option<&Driver>,
- ) -> Result<Classification> {
- Ok(Classification::Ambiguous(Vec::new()))
+ ) -> Result<(Classification, Option<Response>)> {
+ Ok((Classification::Ambiguous(Vec::new()), None))
}
}