Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(prefill-router): do not score an earlier exchange for a text-less user turn (#843)
7konigsberg9ce208bea163
diff --git a/crates/prefill-router/src/algorithm.rs b/crates/prefill-router/src/algorithm.rsindex b0ed4efa9..debe19f05 100644--- a/crates/prefill-router/src/algorithm.rs+++ b/crates/prefill-router/src/algorithm.rs@@ -10,7 +10,7 @@ use async_trait::async_trait;use libsy::{AffinityRouter, Algorithm, Classifier, Driver, Event, LibsyError, Processor, RoutingOutcome,};-use switchyard_protocol::{ModelId, Request, Role};+use switchyard_protocol::{ContentBlock, Message, ModelId, Request, Role};use tokio::sync::Mutex;use crate::{@@ -106,6 +106,37 @@ impl PrefillRouterAlgo {}}+/// The prompt to score: the newest user message that has text of its own.+/// A user message with no text (whitespace only, or an attachment with no+/// caption) is skipped, but once one has been skipped the search stops at the+/// first assistant message, so an earlier exchange's prompt is never scored as+/// the current request. A user message carrying tool results is tool activity+/// inside the current exchange, not a skipped turn, and is walked past as+/// before. `None` leaves the choice to the default target.+fn latest_user_prompt(messages: &[Message]) -> Option<String> {+ let mut skipped = false;+ for message in messages.iter().rev() {+ if message.role == Role::User {+ if let Some(text) = message+ .text_content("\n")+ .filter(|text| !text.trim().is_empty())+ {+ return Some(text);+ }+ if !message+ .content+ .iter()+ .any(|block| matches!(block, ContentBlock::ToolResult(_)))+ {+ skipped = true;+ }+ } else if message.role == Role::Assistant && skipped {+ return None;+ }+ }+ None+}+#[async_trait]impl Algorithm for PrefillRouterAlgo {fn name(&self) -> &str {@@ -138,15 +169,7 @@ impl Algorithm for PrefillRouterAlgo {));}- let target = match request- .llm_request- .messages- .iter()- .rev()- .filter(|message| message.role == Role::User)- .filter_map(|message| message.text_content("\n"))- .find(|text| !text.trim().is_empty())- {+ let target = match latest_user_prompt(&request.llm_request.messages) {Some(prompt) => {// Wait asynchronously before occupying a blocking worker with model inference.let mut router = Arc::clone(&self.router).lock_owned().await;diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rsindex f1b338cf9..2aad5fc7c 100644--- a/crates/prefill-router/tests/unit/algorithm.rs+++ b/crates/prefill-router/tests/unit/algorithm.rs@@ -111,6 +111,97 @@ fn request(messages: Vec<Message>) -> Request {}}+#[tokio::test]+async fn empty_trailing_turn_does_not_score_the_previous_exchange() -> libsy::Result<()> {+ let (forward, calls, prompts) = forward();+ let route: Arc<dyn Algorithm> = Arc::new(+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {+ LibsyError::AlgorithmError {+ message: error.to_string(),+ }+ })?,+ );+ // The newest user turn has no text; the turn before the assistant's reply+ // belongs to a finished exchange and must not be scored in its place.+ let routed = selected(+ route,+ request(vec![+ Message::text(Role::User, "initial task"),+ Message::text(Role::Assistant, "done"),+ Message::text(Role::User, " "),+ ]),+ )+ .await?;+ assert_eq!(routed, "small");+ assert_eq!(calls.load(Ordering::Relaxed), 0);+ assert!(prompts.lock().expect("test prompts lock").is_empty());+ Ok(())+}++#[tokio::test]+async fn empty_user_turn_beside_the_person_still_scores_the_person() -> libsy::Result<()> {+ let (forward, _calls, prompts) = forward();+ let route: Arc<dyn Algorithm> = Arc::new(+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {+ LibsyError::AlgorithmError {+ message: error.to_string(),+ }+ })?,+ );+ // No assistant reply between the two: the empty turn is skipped and the+ // person's text in the same exchange is scored.+ selected(+ route,+ request(vec![+ Message::text(Role::User, "initial task"),+ Message::text(Role::User, " "),+ ]),+ )+ .await?;+ assert_eq!(+ prompts.lock().expect("test prompts lock").as_slice(),+ ["initial task"]+ );+ Ok(())+}++#[tokio::test]+async fn tool_result_after_the_person_is_walked_past_not_skipped() -> libsy::Result<()> {+ let (forward, _calls, prompts) = forward();+ let route: Arc<dyn Algorithm> = Arc::new(+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {+ LibsyError::AlgorithmError {+ message: error.to_string(),+ }+ })?,+ );+ // A tool result is a text-less user message inside the current exchange;+ // it must not count as a skipped turn that ends the search at the tool call.+ selected(+ route,+ request(vec![+ Message::text(Role::User, "follow-up task"),+ Message::text(Role::Assistant, "calling a tool"),+ Message {+ role: Role::User,+ content: vec![ContentBlock::ToolResult(ToolResult {+ tool_call_id: "call-1".to_string(),+ content: vec![ContentBlock::Text {+ text: "tool output".to_string(),+ }],+ is_error: None,+ })],+ },+ ]),+ )+ .await?;+ assert_eq!(+ prompts.lock().expect("test prompts lock").as_slice(),+ ["follow-up task"]+ );+ Ok(())+}+#[tokio::test]async fn routes_on_latest_user_text_and_reuses_decision_for_tool_steps() -> libsy::Result<()> {let (forward, calls, prompts) = forward();