Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): validate classifier scores and drain streamed verdicts (#104)
Elyas Mehtabuddin1e51ccbc93b0
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex e8d65fd56..53bb539c4 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -200,16 +200,24 @@ impl Algorithm for LlmClassifier {classify_decision,).await?;- let score = classify_response+ // Drain a streamed classifier response to its aggregate so a streamed+ // score is read instead of silently dropped. Keep only a valid+ // probability in [0.0, 1.0]: NaN, ±inf, and out-of-range values parse+ // as f64 but are not usable scores, so they collapse to None and fail+ // open to strong below rather than being treated as a real verdict.+ let classify_agg = classify_response.llm_response- .as_agg()- .map(completion_text)- .unwrap_or_default()+ .into_agg()+ .await+ .map_err(|error| LibsyError::client_call(&self.classifier_model, error))?;+ let score = completion_text(&classify_agg).trim().parse::<f64>()- .ok();+ .ok()+ .filter(|s| (0.0..=1.0).contains(s));- // 2. Route: pick strong/weak. Fail open — an unparseable score routes strong.+ // 2. Route: pick strong/weak. Fail open — an unparseable or out-of-range+ // score routes strong.let (tier, model) = match score {Some(s) if s >= self.threshold => (ClassifierTier::Strong, self.strong_model.clone()),Some(_) => (ClassifierTier::Weak, self.weak_model.clone()),@@ -236,7 +244,8 @@ impl Algorithm for LlmClassifier {#[cfg(test)]mod tests {use super::*;- use crate::{LlmResponse, LlmTarget, Response, RoutedLlmClient};+ use crate::{LlmResponse, LlmResponseChunk, LlmTarget, Response, RoutedLlmClient};+ use futures::StreamExt;use parking_lot::Mutex;use switchyard_protocol::text_response;@@ -246,6 +255,9 @@ mod tests {struct ScoringClient {classifier_model: String,score: String,+ /// When true, the classifier target returns its score as a live stream+ /// rather than a buffered aggregate, exercising the drain-the-stream path.+ stream: bool,seen: Arc<Mutex<Vec<Request>>>,}@@ -258,14 +270,26 @@ mod tests {decision: Arc<dyn Decision>,) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {let name = decision.selected_model().to_string();- let completion = if name == self.classifier_model {+ let is_classifier = name == self.classifier_model;+ let completion = if is_classifier {self.score.clone()} else {format!("answer from {name}")};self.seen.lock().push(request);+ // A streaming classifier target emits its score as a live TextDelta+ // stream; the algorithm must drain it (into_agg) to read the score.+ let llm_response = if is_classifier && self.stream {+ let chunks = vec![Ok(LlmResponseChunk::TextDelta {+ index: 0,+ text: completion,+ })];+ LlmResponse::Stream(futures::stream::iter(chunks).boxed())+ } else {+ LlmResponse::Agg(text_response(None, completion))+ };Ok(Response {- llm_response: LlmResponse::Agg(text_response(None, completion)),+ llm_response,metadata: None,})}@@ -273,10 +297,27 @@ mod tests {/// Build a classifier algo whose three targets share a scoring client.fn algo(threshold: f64, score: &str) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {+ build_algo(threshold, score, false)+ }++ /// Like [`algo`], but the classifier target returns its score as a live stream.+ fn algo_streaming(+ threshold: f64,+ score: &str,+ ) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {+ build_algo(threshold, score, true)+ }++ fn build_algo(+ threshold: f64,+ score: &str,+ stream: bool,+ ) -> Result<(LlmClassifier, Arc<Mutex<Vec<Request>>>)> {let seen = Arc::new(Mutex::new(Vec::new()));let client = Arc::new(ScoringClient {classifier_model: "router/classifier".to_string(),score: score.to_string(),+ stream,seen: Arc::clone(&seen),}) as Arc<dyn RoutedLlmClient>;let target = |name: &str| LlmTarget {@@ -409,4 +450,118 @@ mod tests {.unwrap_or_default();assert!(error.contains("threshold must be between 0 and 1"));}++ #[tokio::test]+ async fn out_of_range_score_defaults_to_strong() -> Result<()> {+ // "NaN" parses as an f64 but is not a usable probability, so it must fail+ // open to strong rather than be treated as a below-threshold verdict and+ // silently downgraded to weak.+ let (algo, _) = algo(0.5, "NaN")?;+ let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from frontier/model"+ );+ let routed = as_classifier(&trace[1])?;+ assert_eq!(routed.tier, Some(ClassifierTier::Strong));+ assert_eq!(routed.score, None);+ Ok(())+ }++ #[tokio::test]+ async fn below_range_score_defaults_to_strong() -> Result<()> {+ // "-0.1" parses as an f64 but is below the valid probability range, so it+ // must fail open to strong. The old code compared it against the threshold+ // (-0.1 < 0.5) and routed weak.+ let (algo, _) = algo(0.5, "-0.1")?;+ let (trace, response) = orch(algo).run(Context::default(), request("hi")).await?;+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from frontier/model"+ );+ let routed = as_classifier(&trace[1])?;+ assert_eq!(routed.tier, Some(ClassifierTier::Strong));+ assert_eq!(routed.score, None);+ Ok(())+ }++ #[tokio::test]+ async fn streamed_score_below_threshold_routes_weak() -> Result<()> {+ // A streaming classifier target must have its score drained and read, not+ // dropped — otherwise every streamed verdict falls open to strong+ // regardless of value.+ let (algo, _) = algo_streaming(0.5, "0.2")?;+ let (trace, response) = orch(algo)+ .run(Context::default(), request("say hello"))+ .await?;+ assert_eq!(+ response+ .llm_response+ .as_agg()+ .map(completion_text)+ .unwrap_or_default(),+ "answer from cheap/model"+ );+ let routed = as_classifier(&trace[1])?;+ assert_eq!(routed.tier, Some(ClassifierTier::Weak));+ assert_eq!(routed.score, Some(0.2));+ Ok(())+ }++ #[tokio::test]+ async fn classifier_stream_error_propagates_not_swallowed() -> Result<()> {+ // A classifier stream that fails mid-drain must surface as an error, not be+ // swallowed into a None score that silently routes strong. Guards the+ // into_agg() error branch: swapping the `?` for `.ok()` would fail open on a+ // real stream failure, and this test would catch it.+ struct ErroringStreamClient;+ #[async_trait]+ impl RoutedLlmClient for ErroringStreamClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ _decision: Arc<dyn Decision>,+ ) -> std::result::Result<Response, switchyard_protocol::LlmClientError> {+ let chunks = vec![Err(switchyard_protocol::LlmClientError::General(+ "classifier stream failed".to_string(),+ ))];+ Ok(Response {+ llm_response: LlmResponse::Stream(futures::stream::iter(chunks).boxed()),+ metadata: None,+ })+ }+ }+ let client = Arc::new(ErroringStreamClient) as Arc<dyn RoutedLlmClient>;+ let target = |name: &str| LlmTarget {+ semantic_name: name.to_string(),+ llm_client: Some(client.clone()),+ };+ let target_set = LlmTargetSet::new(vec![+ target("router/classifier"),+ target("frontier/model"),+ target("cheap/model"),+ ]);+ let algo = LlmClassifier::new(+ "router/classifier",+ "frontier/model",+ "cheap/model",+ 0.5,+ target_set,+ )?;+ match orch(algo).run(Context::default(), request("hi")).await {+ Err(LibsyError::ClientCall { target, .. }) => assert_eq!(target, "router/classifier"),+ Err(other) => panic!("expected a ClientCall error, got: {other}"),+ Ok(_) => panic!("expected the classifier stream error to propagate, not fail open"),+ }+ Ok(())+ }}