Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): restate the routing instruction after windowed conversation (#520)

GDRS7c0b46538825
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index d2a12582b..fc680dd60 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -34,6 +34,17 @@ const SCHEMA_TEMPLATE: &str = include_str!("../prompts/capability-classifier/sch
/// Telemetry label for this algorithm's spans, metrics, and logs.
const ALGORITHM_NAME: &str = "llm_task_classifier";
+/// Restates the task after the conversation the judge is asked to route.
+///
+/// The rubric leads the request, which works while the payload is a single short
+/// message. Once `recent_turn_window` includes real conversation, those instructions
+/// sit far from the generation point and the judge sometimes *answers* the
+/// conversation instead of classifying it — the reply then fails to parse, the
+/// verdict is unavailable, and the turn silently falls back. Repeating the task at
+/// the end is what reliably prevents that.
+const TRAILING_ROUTING_INSTRUCTION: &str =
+ "Route the conversation above. Output ONLY the routing JSON object, nothing else.";
+
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct TaskClassifierVerdict {
@@ -157,10 +168,20 @@ impl ClassifierInput for TaskInput {
fn build_messages(&self, _state: &State, request: &Request) -> Vec<Message> {
// The default preserves the whole-task anchor and latest user update. A
// configured window widens that to the surrounding conversation.
- match self.recent_turn_window {
+ let mut messages = match self.recent_turn_window {
Some(window) => trim_messages(&request.llm_request.messages, window),
None => task_messages(&request.llm_request.messages),
+ };
+ // Only the windowed path carries assistant turns and tool traffic for the judge
+ // to be distracted by. The default path is user task messages only — the anchor
+ // and the latest follow-up — so there is nothing there to outrank.
+ if self.recent_turn_window.is_some() {
+ messages.push(Message::text(
+ Role::User,
+ TRAILING_ROUTING_INSTRUCTION.to_string(),
+ ));
}
+ messages
}
}
@@ -1593,6 +1614,50 @@ mod tests {
);
}
+ #[test]
+ fn a_window_restates_the_routing_instruction_last() -> Result<()> {
+ let contents = judged_contents(2)?;
+
+ // Last, not merely present: the instruction only works in final position,
+ // after the conversation content it is meant to outrank.
+ assert_eq!(
+ contents.last().map(String::as_str),
+ Some(TRAILING_ROUTING_INSTRUCTION)
+ );
+ Ok(())
+ }
+
+ #[test]
+ fn the_default_path_is_left_unchanged() -> Result<()> {
+ // No window means no conversation to be distracted by, so the default
+ // request shape stays exactly as it was.
+ let judge = capability_judge(None)?;
+ let request = Request {
+ llm_request: LlmRequest {
+ messages: vec![Message::text(Role::User, "the task")],
+ ..LlmRequest::default()
+ },
+ raw_request: None,
+ metadata: None,
+ };
+
+ let built = judge.build_request(&State::default(), &request);
+
+ // The task message alone; the rubric reaches the judge as an instruction block
+ // rather than a message, so nothing was dropped by it not being counted here.
+ assert_eq!(built.llm_request.messages.len(), 1);
+ assert!(!built.llm_request.instructions.is_empty());
+ assert!(
+ !built
+ .llm_request
+ .messages
+ .iter()
+ .filter_map(|message| message.text_content("\n"))
+ .any(|text| text.contains(TRAILING_ROUTING_INSTRUCTION))
+ );
+ Ok(())
+ }
+
#[test]
fn capability_judge_builds_a_structured_request() -> Result<()> {
let judge = capability_judge(None)?;