Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
refactor(libsy): extract load_judge_config and drop reasoning fallback in judge verdict parsing (#204)
Lin Jia6b9a168cf6b8
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 58678d446..06dac2112 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -11,10 +11,11 @@ use std::sync::Arc;use async_trait::async_trait;use serde::Deserialize;-use serde_json::Value;-use switchyard_protocol::{InstructionBlock, LlmRequest, Message, OutputParams, Role};+use switchyard_protocol::{LlmRequest, Message, OutputParams, Role};-use super::util::{AffinityRouter, Judge, JudgeClassifier, JudgeConfig, JudgePolicy};+use super::util::{+ load_judge_config, AffinityRouter, Judge, JudgeClassifier, JudgeConfig, JudgePolicy,+};use super::{DefaultTarget, FallThrough};use crate::{Algorithm, Classification, Classifier, Context, Driver, LibsyError, LlmTarget, LlmTargetSet,@@ -95,7 +96,7 @@ impl Judge for CapabilityJudge {fn build_request(&self, _state: &State, request: &Request) -> Request {// Task-based routing judges the newest user message alone. A configured// window widens that to the surrounding conversation.- let messages = match self.recent_turn_window {+ let mut messages = match self.recent_turn_window {Some(window) => trim_messages(&request.llm_request.messages, window),None => request.llm_request@@ -107,13 +108,13 @@ impl Judge for CapabilityJudge {.into_iter().collect::<Vec<_>>(),};+ messages.insert(+ 0,+ Message::text(Role::System, self.config.system_prompt.clone()),+ );Request {llm_request: LlmRequest {model: request.llm_request.model.clone(),- instructions: vec![InstructionBlock {- role: Role::System,- content: Message::text(Role::System, self.config.system_prompt.clone()).content,- }],messages,output: OutputParams {response_format: self.config.response_schema.clone(),@@ -325,27 +326,9 @@ impl LlmTaskClassifier {})}+ /// Loads the judge configuration from the packaged prompt and schema.fn load_judge_config() -> Result<JudgeConfig> {- // The response schema is rendered into the prompt and sent as structured-output metadata.- // One asset therefore defines both the instruction contract and provider enforcement.- let response_schema: Value =- serde_json::from_str(SCHEMA_TEMPLATE).map_err(|error| LibsyError::AlgorithmError {- message: format!("capability response schema is invalid: {error}"),- })?;- let prompt_schema = response_schema- .pointer("/json_schema/schema")- .ok_or_else(|| LibsyError::AlgorithmError {- message: "capability response schema has no json_schema.schema".to_string(),- })?;- let prompt_schema = serde_json::to_string_pretty(prompt_schema).map_err(|error| {- LibsyError::AlgorithmError {- message: format!("capability prompt schema could not be rendered: {error}"),- }- })?;- Ok(JudgeConfig {- system_prompt: PROMPT_TEMPLATE.replace("{{RESPONSE_SCHEMA}}", &prompt_schema),- response_schema: Some(response_schema),- })+ load_judge_config(PROMPT_TEMPLATE, SCHEMA_TEMPLATE)}}@@ -414,6 +397,7 @@ mod tests {use std::sync::Arc;use parking_lot::Mutex;+ use serde_json::Value;use super::*;use switchyard_protocol::{completion_text, text_response, LlmClientError, Metadata};@@ -863,13 +847,7 @@ mod tests {let judge_request = judge.build_request(&State::default(), &request);assert_eq!(judge_request.llm_request.model, request.llm_request.model);- assert_eq!(judge_request.llm_request.instructions.len(), 1);- assert_eq!(judge_request.llm_request.instructions[0].role, Role::System);- assert_eq!(- judge_request.llm_request.instructions[0].content,- Message::text(Role::System, judge.config.system_prompt.clone()).content- );- assert_eq!(judge_request.llm_request.messages.len(), 1);+ assert_eq!(judge_request.llm_request.messages.len(), 2);let contents = judge_request.llm_request.messagesdiff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rsindex f86e23f9a..e21dff5a3 100644--- a/crates/libsy/src/algorithms/util.rs+++ b/crates/libsy/src/algorithms/util.rs@@ -9,6 +9,6 @@ mod subagent;pub(crate) mod tool_signals;pub use affinity::AffinityRouter;-pub(crate) use llm_judge::{Judge, JudgeClassifier, JudgeConfig, JudgePolicy};+pub(crate) use llm_judge::{load_judge_config, Judge, JudgeClassifier, JudgeConfig, JudgePolicy};pub use prompts::{append_note, SystemPromptProcessor, TargetPrompts};pub use subagent::SubagentOverride;diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex e876debe4..0654cfe91 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -143,16 +143,43 @@ where}}-fn parse_json_verdict<T: DeserializeOwned>(response: &AggLlmResponse) -> Result<T> {- // Providers sometimes wrap otherwise valid JSON in a Markdown fence.- let completion = completion_text(response);- serde_json::from_str(strip_json_fence(completion.trim())).map_err(|err| {+/// Builds a [`JudgeConfig`] from a prompt template and a JSON schema template.+///+/// `schema_template` must be a `{ "type": "json_schema", "json_schema": { "schema": ... } }`+/// object; the inner `schema` is substituted into the `{{RESPONSE_SCHEMA}}` placeholder in+/// `prompt_template`.+pub(crate) fn load_judge_config(+ prompt_template: &str,+ schema_template: &str,+) -> Result<JudgeConfig> {+ let response_schema: Value =+ serde_json::from_str(schema_template).map_err(|error| LibsyError::AlgorithmError {+ message: format!("response schema is invalid: {error}"),+ })?;+ let prompt_schema = response_schema+ .pointer("/json_schema/schema")+ .ok_or_else(|| LibsyError::AlgorithmError {+ message: "response schema has no json_schema.schema".to_string(),+ })?;+ let prompt_schema = serde_json::to_string_pretty(prompt_schema).map_err(|error| {LibsyError::AlgorithmError {- message: format!(- "judge reply did not parse as {}: {err}",- std::any::type_name::<T>()- ),+ message: format!("prompt schema could not be rendered: {error}"),}+ })?;+ Ok(JudgeConfig {+ system_prompt: prompt_template.replace("{{RESPONSE_SCHEMA}}", &prompt_schema),+ response_schema: Some(response_schema),+ })+}++fn parse_json_verdict<T: DeserializeOwned>(response: &AggLlmResponse) -> Result<T> {+ // Providers sometimes wrap otherwise valid JSON in a Markdown fence.+ let reply = completion_text(response);+ serde_json::from_str(strip_json_fence(reply.trim())).map_err(|err| LibsyError::AlgorithmError {+ message: format!(+ "judge reply did not parse as {}: {err}",+ std::any::type_name::<T>()+ ),})}@@ -193,7 +220,7 @@ mod tests {use futures::StreamExt;use serde::Deserialize;- use switchyard_protocol::{text_request, text_response, LlmClientError};+ use switchyard_protocol::{text_request, text_response, ContentBlock, LlmClientError};use crate::{LlmResponse, LlmResponseChunk, Response, Score, Step};@@ -252,6 +279,28 @@ mod tests {}}+ #[test]+ fn the_verdict_is_read_from_the_completion() -> Result<()> {+ // A judge's reasoning is not its answer: only `content` carries the verdict, so a+ // reply that never reached one — a run truncated mid-thought — is an error rather+ // than a guess.+ let mut response = text_response(None, VERDICT);+ if let Some(output) = response.outputs.first_mut() {+ output.content.insert(+ 0,+ ContentBlock::Reasoning {+ text: r#"{"ok":false}"#.to_string(),+ signature: None,+ },+ );+ }+ let parsed: TestVerdict = parse_json_verdict(&response)?;+ assert_eq!(parsed, TestVerdict { ok: true });++ assert!(parse_json_verdict::<TestVerdict>(&text_response(None, "still thinking")).is_err());+ Ok(())+ }+fn buffered(completion: &str) -> Response {Response {llm_response: LlmResponse::Agg(text_response(None, completion)),@@ -308,7 +357,7 @@ mod tests {classifier.score(&mut state, &mut request, Some(&driver)),serve);- selected(classification?)+ selected(classification?.0)}#[tokio::test]