Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(routing): exclude tool results from default classifier input (#751)

Elyas Mehtabuddin5c3f32dd37ae
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index c18086e7f..12d9241d0 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -150,14 +150,35 @@ fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize {
/// Keeps the opening task and the latest user follow-up when they differ.
fn task_messages(messages: &[Message]) -> Vec<Message> {
- let mut user_messages = messages.iter().filter(|message| message.role == Role::User);
+ // Decoders also use the user role for tool results. Select ordinary user content
+ // first, so a tool result cannot replace the opening task or latest follow-up.
+ let is_task_content = |block: &ContentBlock| {
+ !matches!(
+ block,
+ ContentBlock::ToolCall(_)
+ | ContentBlock::ToolResult(_)
+ | ContentBlock::Reasoning { .. }
+ )
+ };
+ let mut user_messages = messages.iter().filter(|message| {
+ message.role == Role::User && message.content.iter().any(is_task_content)
+ });
let Some(opening_task) = user_messages.next() else {
return Vec::new();
};
- match user_messages.next_back() {
- Some(latest_follow_up) => vec![opening_task.clone(), latest_follow_up.clone()],
- None => vec![opening_task.clone()],
- }
+ [Some(opening_task), user_messages.next_back()]
+ .into_iter()
+ .flatten()
+ .map(|message| Message {
+ role: Role::User,
+ content: message
+ .content
+ .iter()
+ .filter(|block| is_task_content(block))
+ .cloned()
+ .collect(),
+ })
+ .collect()
}
/// Selects the task messages shown to capability and custom-schema classifiers.
@@ -1398,6 +1419,41 @@ mod tests {
}
}
+ #[test]
+ fn default_task_input_keeps_user_content_around_tool_results() {
+ let mut result = tool_result("call-1");
+ result.role = Role::User;
+ let mut mixed = result.clone();
+ mixed.content.push(ContentBlock::Text {
+ text: "latest follow-up".to_string(),
+ });
+ let input = TaskInput {
+ recent_turn_window: None,
+ };
+ let mut request = Request {
+ llm_request: LlmRequest {
+ messages: vec![
+ result.clone(),
+ Message::text(Role::User, "initial task"),
+ tool_call("call-1"),
+ mixed,
+ result.clone(),
+ ],
+ ..LlmRequest::default()
+ },
+ ..Request::default()
+ };
+ assert_eq!(
+ input.build_messages(&State::default(), &request),
+ vec![
+ Message::text(Role::User, "initial task"),
+ Message::text(Role::User, "latest follow-up"),
+ ]
+ );
+ request.llm_request.messages = vec![result];
+ assert!(input.build_messages(&State::default(), &request).is_empty());
+ }
+
/// A count-based window can begin on a tool result, which leaves the call that
/// introduced its id outside the window and the classifier history invalid.
#[test]
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 661c59f25..76ecbc1a1 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -140,6 +140,27 @@ async fn upstream_chat(
}
let model = body["model"].as_str().unwrap_or("unknown").to_string();
+ if model == "model/classifier" {
+ let mut pending = HashSet::new();
+ for message in body["messages"].as_array().into_iter().flatten() {
+ for call in message["tool_calls"].as_array().into_iter().flatten() {
+ if let Some(id) = call["id"].as_str() {
+ pending.insert(id);
+ }
+ }
+ if message["role"] == "tool"
+ && !pending.remove(message["tool_call_id"].as_str().unwrap_or_default())
+ {
+ return (
+ StatusCode::BAD_REQUEST,
+ Json(json!({"error": {
+ "message": "tool result has no preceding assistant tool call"
+ }})),
+ )
+ .into_response();
+ }
+ }
+ }
if prompt == "retry-once"
&& calls
.lock()
@@ -2586,6 +2607,113 @@ prompt = "CUSTOM STAGE"
Ok(())
}
+#[tokio::test]
+async fn classifier_task_input_excludes_tool_results_across_request_formats() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let app = build_switchyard_router(load_test_config(&format!(
+ r#"
+schema_version = 1
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+max_retries = 0
+[targets.judge]
+id = "model/classifier"
+llm_client = "upstream"
+[targets.strong]
+id = "model/strong"
+llm_client = "upstream"
+[targets.weak]
+id = "model/weak"
+llm_client = "upstream"
+[routes.task]
+id = "task"
+type = "llm_classifier"
+classifier_target = "judge"
+strong_target = "strong"
+weak_target = "weak"
+base_threshold = 0.5
+classify_trigger = "every_request"
+[routes.window]
+id = "window"
+type = "llm_classifier"
+classifier_target = "judge"
+strong_target = "strong"
+weak_target = "weak"
+base_threshold = 0.5
+classify_trigger = "every_request"
+recent_turn_window = 1
+"#,
+ base_url = upstream.base_url
+ ))?);
+ let chat = json!({"messages": [
+ {"role": "user", "content": "Read the file."},
+ {"role": "assistant", "content": null, "tool_calls": [{
+ "id": "call_read", "type": "function", "function": {"name": "read_file", "arguments": "{}"}
+ }]},
+ {"role": "tool", "tool_call_id": "call_read", "content": "File contents."}
+ ]});
+ for (route, endpoint, mut body) in [
+ ("task", "/v1/chat/completions", chat.clone()),
+ (
+ "task",
+ "/v1/messages",
+ json!({"messages": [
+ {"role": "user", "content": "Read the file."},
+ {"role": "assistant", "content": [{"type": "tool_use", "id": "call_read", "name": "read_file", "input": {}}]},
+ {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_read", "content": "File contents."}]}
+ ]}),
+ ),
+ (
+ "task",
+ "/v1/responses",
+ json!({"input": [
+ {"role": "user", "content": "Read the file."},
+ {"type": "function_call", "call_id": "call_read", "name": "read_file", "arguments": "{}"},
+ {"type": "function_call_output", "call_id": "call_read", "output": "File contents."}
+ ]}),
+ ),
+ ("window", "/v1/chat/completions", chat),
+ ] {
+ upstream.calls.lock().await.clear();
+ body["model"] = json!(route);
+ let response = send(&app, "POST", endpoint, Some(body)).await?;
+ assert_eq!(response.status, StatusCode::OK, "{}", response.text()?);
+ assert_eq!(
+ response.headers["x-model-router-selected-model"], "model/weak",
+ "{endpoint}"
+ );
+ let calls = upstream.calls.lock().await;
+ assert_eq!(calls.len(), 2);
+ let judge_roles: Vec<_> = calls[0]["messages"]
+ .as_array()
+ .ok_or("missing judge messages")?
+ .iter()
+ .map(|message| message["role"].clone())
+ .collect();
+ assert_eq!(
+ judge_roles,
+ if route == "task" {
+ json!(["system", "user"])
+ } else {
+ json!(["system", "user", "assistant", "tool", "user"])
+ }
+ .as_array()
+ .ok_or("missing expected roles")?
+ .clone()
+ );
+ assert_eq!(calls[1]["messages"][1]["tool_calls"][0]["id"], "call_read");
+ assert_eq!(calls[1]["messages"][2]["tool_call_id"], "call_read");
+ }
+ let stats = send(&app, "GET", "/v1/stats", None).await?.json()?;
+ assert_eq!(stats["classifier"]["total_requests"], 4);
+ assert_eq!(
+ stats["classifier"]["models"]["model/classifier"]["errors"],
+ 0
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn accepted_escalation_response_is_logged_once_as_the_final_answer() -> TestResult {
let upstream = MockUpstream::start().await?;
diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md
index d009665e8..f7839fa0f 100644
--- a/docs/routing_algorithms/llm_classifier_routing.md
+++ b/docs/routing_algorithms/llm_classifier_routing.md
@@ -227,8 +227,10 @@ The deterministic policy applies `base_threshold` and `threshold_step` after
generation.
Without affinity, the runtime judges every request. By default, it sends the
-opening task and the latest user follow-up when they differ. Set
-`recent_turn_window` when intervening conversation context affects the forecast.
+opening task and the latest user follow-up when they differ, excluding tool calls,
+tool results, and reasoning. It keeps ordinary user content from messages that
+also contain tool results. Set `recent_turn_window` when intervening conversation
+context affects the forecast.
If a client sends only a follow-up fragment without the opening task, enable
affinity or include the task history. Threshold tuning changes routing policy;
it cannot recover missing task context.