Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): validate classifier scores and drain streamed verdicts (#104)

Elyas Mehtabuddin1e51ccbc93b0
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index e8d65fd56..53bb539c4 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -200,16 +200,24 @@ impl Algorithm for LlmClassifier {
classify_decision,
)
.await?;
- let score = classify_response
+ // Drain a streamed classifier response to its aggregate so a streamed
+ // score is read instead of silently dropped. Keep only a valid
+ // probability in [0.0, 1.0]: NaN, ±inf, and out-of-range values parse
+ // as f64 but are not usable scores, so they collapse to None and fail
+ // open to strong below rather than being treated as a real verdict.
+ let classify_agg = classify_response
.llm_response
- .as_agg()
- .map(completion_text)
- .unwrap_or_default()
+ .into_agg()
+ .await
+ .map_err(|error| LibsyError::client_call(&self.classifier_model, error))?;
+ let score = completion_text(&classify_agg)
.trim()
.parse::<f64>()
- .ok();
+ .ok()
+ .filter(|s| (0.0..=1.0).contains(s));
- // 2. Route: pick strong/weak. Fail open — an unparseable score routes strong.
+ // 2. Route: pick strong/weak. Fail open — an unparseable or out-of-range
+ // score routes strong.
let (tier, model) = match score {
Some(s) if s >= self.threshold => (ClassifierTier::Strong, self.strong_model.clone()),
Some(_) => (ClassifierTier::Weak, self.weak_model.clone()),
@@ -236,7 +244,8 @@ impl Algorithm for LlmClassifier {
#[cfg(test)]
mod tests {
use super::*;
- use crate::{LlmResponse, LlmTarget, Response, RoutedLlmClient};
+ use crate::{LlmResponse, LlmResponseChunk, LlmTarget, Response, RoutedLlmClient};
+ use futures::StreamExt;
use parking_lot::Mutex;
use switchyard_protocol::text_response;
@@ -246,6 +255,9 @@ mod tests {
struct ScoringClient {
classifier_model: String,
score: String,
+ /// When true, the classifier target returns its score as a live stream
+ /// rather than a buffered aggregate, exercising the drain-the-stream path.
+ stream: bool,
seen: Arc<Mutex<Vec<Request>>>,
}
@@ -258,14 +270,26 @@ mod tests {
decision: Arc<dyn Decision>,
) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
let name = decision.selected_model().to_string();
- let completion = if name == self.classifier_model {
+ let is_classifier = name == self.classifier_model;
+ let completion = if is_classifier {
self.score.clone()
} else {
format!("answer from {name}")
};
self.seen.lock().push(request);
+ // A streaming classifier target emits its score as a live TextDelta
+ // stream; the algorithm must drain it (into_agg) to read the score.
+ let llm_response = if is_classifier && self.stream {
+ let chunks = vec![Ok(LlmResponseChunk::TextDelta {
+ index: 0,
+ text: completion,
+ })];
+ LlmResponse::Stream(futures::stream::iter(chunks).boxed())
+ } else {
+ LlmResponse::Agg(text_response(None, completion))
+ };
Ok(Response {
- llm_response: LlmResponse::Agg(text_response(None, completion)),
+ llm_response,
metadata: None,
})
}
@@ -273,10 +297,27 @@ mod tests {
/// Build a classifier algo whose three targets share a scoring client.
fn algo(threshold: f64, score: &str) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
+ build_algo(threshold, score, false)
+ }
+
+ /// Like [`algo`], but the classifier target returns its score as a live stream.
+ fn algo_streaming(
+ threshold: f64,
+ score: &str,
+ ) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
+ build_algo(threshold, score, true)
+ }
+
+ fn build_algo(
+ threshold: f64,
+ score: &str,
+ stream: bool,
+ ) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {
let seen = Arc::new(Mutex::new(Vec::new()));
let client = Arc::new(ScoringClient {
classifier_model: "router/classifier".to_string(),
score: score.to_string(),
+ stream,
seen: Arc::clone(&seen),
}) as Arc<dyn RoutedLlmClient>;
let target = |name: &str| LlmTarget {
@@ -409,4 +450,118 @@ mod tests {
.unwrap_or_default();
assert!(error.contains("threshold must be between 0 and 1"));
}
+
+ #[tokio::test]
+ async fn out_of_range_score_defaults_to_strong() -> Result<()> {
+ // "NaN" parses as an f64 but is not a usable probability, so it must fail
+ // open to strong rather than be treated as a below-threshold verdict and
+ // silently downgraded to weak.
+ let (algo, _) = algo(0.5, "NaN")?;
+ let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from frontier/model"
+ );
+ let routed = as_classifier(&trace[1])?;
+ assert_eq!(routed.tier, Some(ClassifierTier::Strong));
+ assert_eq!(routed.score, None);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn below_range_score_defaults_to_strong() -> Result<()> {
+ // "-0.1" parses as an f64 but is below the valid probability range, so it
+ // must fail open to strong. The old code compared it against the threshold
+ // (-0.1 < 0.5) and routed weak.
+ let (algo, _) = algo(0.5, "-0.1")?;
+ let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from frontier/model"
+ );
+ let routed = as_classifier(&trace[1])?;
+ assert_eq!(routed.tier, Some(ClassifierTier::Strong));
+ assert_eq!(routed.score, None);
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn streamed_score_below_threshold_routes_weak() -> Result<()> {
+ // A streaming classifier target must have its score drained and read, not
+ // dropped — otherwise every streamed verdict falls open to strong
+ // regardless of value.
+ let (algo, _) = algo_streaming(0.5, "0.2")?;
+ let (trace, response) = orch(algo)
+ .run(Context::default(), request("say hello"))
+ .await?;
+ assert_eq!(
+ response
+ .llm_response
+ .as_agg()
+ .map(completion_text)
+ .unwrap_or_default(),
+ "answer from cheap/model"
+ );
+ let routed = as_classifier(&trace[1])?;
+ assert_eq!(routed.tier, Some(ClassifierTier::Weak));
+ assert_eq!(routed.score, Some(0.2));
+ Ok(())
+ }
+
+ #[tokio::test]
+ async fn classifier_stream_error_propagates_not_swallowed() -> Result<()> {
+ // A classifier stream that fails mid-drain must surface as an error, not be
+ // swallowed into a None score that silently routes strong. Guards the
+ // into_agg() error branch: swapping the `?` for `.ok()` would fail open on a
+ // real stream failure, and this test would catch it.
+ struct ErroringStreamClient;
+ #[async_trait]
+ impl RoutedLlmClient for ErroringStreamClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ _decision: Arc<dyn Decision>,
+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {
+ let chunks = vec![Err(switchyard_protocol::LlmClientError::General(
+ "classifier stream failed".to_string(),
+ ))];
+ Ok(Response {
+ llm_response: LlmResponse::Stream(futures::stream::iter(chunks).boxed()),
+ metadata: None,
+ })
+ }
+ }
+ let client = Arc::new(ErroringStreamClient) as Arc<dyn RoutedLlmClient>;
+ let target = |name: &str| LlmTarget {
+ semantic_name: name.to_string(),
+ llm_client: Some(client.clone()),
+ };
+ let target_set = LlmTargetSet::new(vec![
+ target("router/classifier"),
+ target("frontier/model"),
+ target("cheap/model"),
+ ]);
+ let algo = LlmClassifier::new(
+ "router/classifier",
+ "frontier/model",
+ "cheap/model",
+ 0.5,
+ target_set,
+ )?;
+ match orch(algo).run(Context::default(), request("hi")).await {
+ Err(LibsyError::ClientCall { target, .. }) => assert_eq!(target, "router/classifier"),
+ Err(other) => panic!("expected a ClientCall error, got: {other}"),
+ Ok(_) => panic!("expected the classifier stream error to propagate, not fail open"),
+ }
+ Ok(())
+ }
}