Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(libsy): fall back on efficient stream transport errors (#390)
Elias Shiehb851cdacc491
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex ec5be5957..b60ebea63 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -531,13 +531,17 @@ impl Classifier<State> for EscalationClassifier {}) => return Ok((decisive(&self.capable), None)),Err(e) => return Err(e),};- let agg = efficient_response- .llm_response- .into_agg()- .await- .map_err(|e| LibsyError::AlgorithmError {- message: format!("failed to aggregate efficient response: {e}"),- })?;+ // The call resolves when its stream handle arrives; transport can still fail while+ // buffering. Fall back only for that availability failure and keep other errors typed.+ let agg = match efficient_response.llm_response.into_agg().await {+ Ok(agg) => agg,+ Err(LlmClientError::Transport { .. }) => {+ return Ok((decisive(&self.capable), None));+ }+ Err(source) => {+ return Err(LibsyError::client_call(self.efficient.clone(), source));+ }+ };// Append the efficient reply so the judge reads this turn's completed trajectory.let mut judge_request = request.clone();judge_request@@ -938,8 +942,8 @@ mod tests {use super::*;use switchyard_protocol::{- ContentBlock, InstructionBlock, LlmClientError, LlmRequest, Metadata, ToolCall, ToolResult,- completion_text, text_request, text_response,+ ContentBlock, InstructionBlock, LlmClientError, LlmRequest, LlmResponseChunk, Metadata,+ ToolCall, ToolResult, completion_text, text_request, text_response,};use crate::algorithms::util::llm_judge::Judge;@@ -1771,6 +1775,21 @@ mod tests {}}+ /// Returns a stream that emits partial content before failing during aggregation.+ fn streamed_then_error(error: LlmClientError) -> Response {+ Response {+ llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter([+ Ok(LlmResponseChunk::TextDelta {+ index: 0,+ text: "partial".to_string(),+ }+ .into()),+ Err(error),+ ]))),+ metadata: None,+ }+ }+/// Builds a router with escalation enabled (`confirmations=1` latches on the first verdict).fn escalation_router() -> Result<Arc<LlmTaskClassifier>> {Ok(Arc::new(LlmTaskClassifier::new(@@ -1922,4 +1941,71 @@ mod tests {);Ok(())}++ /// A transport failure while buffering efficient must bypass the judge and serve capable.+ #[tokio::test]+ async fn escalation_classifier_falls_back_when_efficient_stream_transport_fails() -> Result<()>+ {+ let router = escalation_router()?;+ let calls = Arc::new(Mutex::new(Vec::new()));+ let serve = {+ let calls = Arc::clone(&calls);+ move |decision: Decision, _request: Request| {+ let calls = Arc::clone(&calls);+ async move {+ let model = decision.selected_model_id().to_string();+ calls.lock().push(model.clone());+ match model.as_str() {+ "efficient" => Ok(streamed_then_error(LlmClientError::Transport {+ source: Box::new(std::io::Error::other("stream disconnected")),+ })),+ "judge" => {+ panic!("the judge must not be consulted after a transport failure")+ }+ _ => Ok(reply("capable answer")),+ }+ }+ }+ };+ let mut request = classify_request();+ request.llm_request.stream = true;++ let result = test_drive(router, request, serve).await;++ assert_eq!(&*calls.lock(), &["efficient", "capable"]);+ let (_, response) = result?;+ assert_eq!(+ response.llm_response.as_agg().map(completion_text),+ Some("capable answer".to_string())+ );+ Ok(())+ }++ /// Non-transport aggregation failures remain typed and do not silently change targets.+ #[tokio::test]+ async fn escalation_classifier_preserves_non_transport_stream_errors() -> Result<()> {+ let router = escalation_router()?;+ let serve = |decision: Decision, _request: Request| async move {+ match decision.selected_model_id().as_str() {+ "efficient" => Ok(streamed_then_error(LlmClientError::InvalidResponse {+ source: Box::new(std::io::Error::other("invalid stream event")),+ })),+ other => panic!("unexpected call to {other}"),+ }+ };+ let mut request = classify_request();+ request.llm_request.stream = true;++ match test_drive(router, request, serve).await {+ Err(LibsyError::ClientCall {+ target,+ source: LlmClientError::InvalidResponse { .. },+ }) => {+ assert_eq!(target, "efficient");+ Ok(())+ }+ Err(other) => panic!("expected InvalidResponse client error, got {other:?}"),+ Ok(_) => panic!("expected stream aggregation to fail"),+ }+ }}