Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(prefill-router): do not score an earlier exchange for a text-less user turn (#843)

7konigsberg9ce208bea163
diff --git a/crates/prefill-router/src/algorithm.rs b/crates/prefill-router/src/algorithm.rs
index b0ed4efa9..debe19f05 100644
--- a/crates/prefill-router/src/algorithm.rs
+++ b/crates/prefill-router/src/algorithm.rs
@@ -10,7 +10,7 @@ use async_trait::async_trait;
use libsy::{
AffinityRouter, Algorithm, Classifier, Driver, Event, LibsyError, Processor, RoutingOutcome,
};
-use switchyard_protocol::{ModelId, Request, Role};
+use switchyard_protocol::{ContentBlock, Message, ModelId, Request, Role};
use tokio::sync::Mutex;
use crate::{
@@ -106,6 +106,37 @@ impl PrefillRouterAlgo {
}
}
+/// The prompt to score: the newest user message that has text of its own.
+/// A user message with no text (whitespace only, or an attachment with no
+/// caption) is skipped, but once one has been skipped the search stops at the
+/// first assistant message, so an earlier exchange's prompt is never scored as
+/// the current request. A user message carrying tool results is tool activity
+/// inside the current exchange, not a skipped turn, and is walked past as
+/// before. `None` leaves the choice to the default target.
+fn latest_user_prompt(messages: &[Message]) -> Option<String> {
+ let mut skipped = false;
+ for message in messages.iter().rev() {
+ if message.role == Role::User {
+ if let Some(text) = message
+ .text_content("\n")
+ .filter(|text| !text.trim().is_empty())
+ {
+ return Some(text);
+ }
+ if !message
+ .content
+ .iter()
+ .any(|block| matches!(block, ContentBlock::ToolResult(_)))
+ {
+ skipped = true;
+ }
+ } else if message.role == Role::Assistant && skipped {
+ return None;
+ }
+ }
+ None
+}
+
#[async_trait]
impl Algorithm for PrefillRouterAlgo {
fn name(&self) -> &str {
@@ -138,15 +169,7 @@ impl Algorithm for PrefillRouterAlgo {
));
}
- let target = match request
- .llm_request
- .messages
- .iter()
- .rev()
- .filter(|message| message.role == Role::User)
- .filter_map(|message| message.text_content("\n"))
- .find(|text| !text.trim().is_empty())
- {
+ let target = match latest_user_prompt(&request.llm_request.messages) {
Some(prompt) => {
// Wait asynchronously before occupying a blocking worker with model inference.
let mut router = Arc::clone(&self.router).lock_owned().await;
diff --git a/crates/prefill-router/tests/unit/algorithm.rs b/crates/prefill-router/tests/unit/algorithm.rs
index f1b338cf9..2aad5fc7c 100644
--- a/crates/prefill-router/tests/unit/algorithm.rs
+++ b/crates/prefill-router/tests/unit/algorithm.rs
@@ -111,6 +111,97 @@ fn request(messages: Vec<Message>) -> Request {
}
}
+#[tokio::test]
+async fn empty_trailing_turn_does_not_score_the_previous_exchange() -> libsy::Result<()> {
+ let (forward, calls, prompts) = forward();
+ let route: Arc<dyn Algorithm> = Arc::new(
+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {
+ LibsyError::AlgorithmError {
+ message: error.to_string(),
+ }
+ })?,
+ );
+ // The newest user turn has no text; the turn before the assistant's reply
+ // belongs to a finished exchange and must not be scored in its place.
+ let routed = selected(
+ route,
+ request(vec![
+ Message::text(Role::User, "initial task"),
+ Message::text(Role::Assistant, "done"),
+ Message::text(Role::User, " "),
+ ]),
+ )
+ .await?;
+ assert_eq!(routed, "small");
+ assert_eq!(calls.load(Ordering::Relaxed), 0);
+ assert!(prompts.lock().expect("test prompts lock").is_empty());
+ Ok(())
+}
+
+#[tokio::test]
+async fn empty_user_turn_beside_the_person_still_scores_the_person() -> libsy::Result<()> {
+ let (forward, _calls, prompts) = forward();
+ let route: Arc<dyn Algorithm> = Arc::new(
+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {
+ LibsyError::AlgorithmError {
+ message: error.to_string(),
+ }
+ })?,
+ );
+ // No assistant reply between the two: the empty turn is skipped and the
+ // person's text in the same exchange is scored.
+ selected(
+ route,
+ request(vec![
+ Message::text(Role::User, "initial task"),
+ Message::text(Role::User, " "),
+ ]),
+ )
+ .await?;
+ assert_eq!(
+ prompts.lock().expect("test prompts lock").as_slice(),
+ ["initial task"]
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn tool_result_after_the_person_is_walked_past_not_skipped() -> libsy::Result<()> {
+ let (forward, _calls, prompts) = forward();
+ let route: Arc<dyn Algorithm> = Arc::new(
+ PrefillRouterAlgo::from_forward(target_set(), forward).map_err(|error| {
+ LibsyError::AlgorithmError {
+ message: error.to_string(),
+ }
+ })?,
+ );
+ // A tool result is a text-less user message inside the current exchange;
+ // it must not count as a skipped turn that ends the search at the tool call.
+ selected(
+ route,
+ request(vec![
+ Message::text(Role::User, "follow-up task"),
+ Message::text(Role::Assistant, "calling a tool"),
+ Message {
+ role: Role::User,
+ content: vec![ContentBlock::ToolResult(ToolResult {
+ tool_call_id: "call-1".to_string(),
+ content: vec![ContentBlock::Text {
+ text: "tool output".to_string(),
+ }],
+ is_error: None,
+ })],
+ },
+ ]),
+ )
+ .await?;
+ assert_eq!(
+ prompts.lock().expect("test prompts lock").as_slice(),
+ ["follow-up task"]
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn routes_on_latest_user_text_and_reuses_decision_for_tool_steps() -> libsy::Result<()> {
let (forward, calls, prompts) = forward();