Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix: support json_object classifier responses (#411)

Elias Shieh7caed52608aa
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index f00b31ccb..7fec82dc3 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -14,7 +14,9 @@ use switchyard_protocol::{ContentBlock, Decision, Message, ModelId, Role};
use super::fall_through::{DefaultTarget, FallThrough};
use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS;
use super::util::affinity::AffinityRouter;
-use super::util::classifier_contract::{ClassifierContract, ClassifierContractConfig};
+use super::util::classifier_contract::{
+ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat,
+};
use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy};
use super::util::llm_judge::{
ClassifierInput, JsonSchemaDecoder, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig,
@@ -261,6 +263,8 @@ struct TaskClassifierConfigWire {
recent_turn_window: Option<usize>,
#[serde(default)]
prompt: Option<String>,
+ #[serde(default)]
+ response_format_type: ClassifierResponseFormat,
#[serde(default = "default_judge_max_output_tokens")]
max_output_tokens: u64,
}
@@ -275,6 +279,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig {
if let Some(prompt) = wire.prompt {
contract = contract.with_prompt(prompt);
}
+ contract = contract.with_response_format_type(wire.response_format_type);
Ok(Self {
base_threshold: wire.base_threshold,
threshold_step: wire.threshold_step,
diff --git a/crates/libsy/src/algorithms/util/classifier_contract.rs b/crates/libsy/src/algorithms/util/classifier_contract.rs
index d3298dcf3..d597ed750 100644
--- a/crates/libsy/src/algorithms/util/classifier_contract.rs
+++ b/crates/libsy/src/algorithms/util/classifier_contract.rs
@@ -9,6 +9,17 @@ use serde_json::{Value, json};
use crate::{LibsyError, Result};
+/// Provider-side structured-output mode used by a classifier judge.
+#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum ClassifierResponseFormat {
+ /// Send the verdict schema through the provider's strict JSON Schema wrapper.
+ #[default]
+ JsonSchema,
+ /// Request a JSON object and enforce the verdict schema locally.
+ JsonObject,
+}
+
/// User-configurable parts of a classifier's prompt and verdict contract.
///
/// Fields are private so new contract settings can be added without breaking Rust struct literals.
@@ -16,6 +27,8 @@ use crate::{LibsyError, Result};
pub struct ClassifierContractConfig {
#[serde(default)]
prompt: Option<String>,
+ #[serde(default)]
+ response_format_type: ClassifierResponseFormat,
}
impl ClassifierContractConfig {
@@ -29,6 +42,20 @@ impl ClassifierContractConfig {
pub fn prompt(&self) -> Option<&str> {
self.prompt.as_deref()
}
+
+ /// Selects the provider-side structured-output mode.
+ pub fn with_response_format_type(
+ mut self,
+ response_format_type: ClassifierResponseFormat,
+ ) -> Self {
+ self.response_format_type = response_format_type;
+ self
+ }
+
+ /// Returns the configured provider-side structured-output mode.
+ pub fn response_format_type(&self) -> ClassifierResponseFormat {
+ self.response_format_type
+ }
}
/// Rendered prompt and response format for one classifier.
@@ -42,8 +69,9 @@ pub(crate) struct ClassifierContract {
impl ClassifierContract {
/// Builds a contract from user settings and packaged defaults.
///
- /// The response format must contain `json_schema.schema` and is retained separately for the
- /// model request. Schemas are never copied into the system prompt.
+ /// The packaged response format must contain `json_schema.schema`. JSON Schema mode retains
+ /// that wrapper for the model request; JSON Object mode moves the schema into the prompt and
+ /// compiles it for local validation.
pub(crate) fn from_config(
config: &ClassifierContractConfig,
default_prompt: &str,
@@ -56,7 +84,31 @@ impl ClassifierContract {
message: format!("response schema is invalid: {error}"),
}
})?;
- Self::from_response_format(prompt_template, response_format, None)
+ let schema = response_format
+ .pointer("/json_schema/schema")
+ .ok_or_else(|| LibsyError::AlgorithmError {
+ message: "response schema has no json_schema.schema".to_string(),
+ })?;
+ match config.response_format_type() {
+ ClassifierResponseFormat::JsonSchema => {
+ Self::from_response_format(prompt_template, response_format, None)
+ }
+ ClassifierResponseFormat::JsonObject => {
+ validate_prompt(prompt_template)?;
+ let validator = compile_schema(schema)?;
+ let rendered_schema = serde_json::to_string_pretty(schema).map_err(|error| {
+ algorithm_error(format!("response schema could not be rendered: {error}"))
+ })?;
+ let system_prompt = format!(
+ "{prompt_template}\n\nReturn exactly one JSON object matching this JSON Schema:\n{rendered_schema}"
+ );
+ Self::from_response_format(
+ &system_prompt,
+ json!({"type": "json_object"}),
+ Some(validator),
+ )
+ }
+ }
}
/// Builds a provider response format around a user-supplied inner JSON Schema.
@@ -88,21 +140,7 @@ impl ClassifierContract {
response_format: Value,
validator: Option<Validator>,
) -> Result<Self> {
- if prompt_template.trim().is_empty() {
- return Err(LibsyError::AlgorithmError {
- message: "classifier prompt must not be empty".to_string(),
- });
- }
- if prompt_template.contains("{{RESPONSE_SCHEMA}}") {
- return Err(LibsyError::AlgorithmError {
- message: "classifier prompt must not include {{RESPONSE_SCHEMA}}; the response schema is sent separately".to_string(),
- });
- }
- response_format
- .pointer("/json_schema/schema")
- .ok_or_else(|| LibsyError::AlgorithmError {
- message: "response schema has no json_schema.schema".to_string(),
- })?;
+ validate_prompt(prompt_template)?;
Ok(Self {
system_prompt: prompt_template.to_string(),
@@ -119,6 +157,11 @@ impl ClassifierContract {
&self.response_format
}
+ /// Whether the provider response must be checked against the compiled schema locally.
+ pub(crate) fn validates_locally(&self) -> bool {
+ self.validator.is_some()
+ }
+
/// Validates a dynamic verdict when this contract carries a runtime schema validator.
pub(crate) fn validate_verdict(&self, verdict: &Value) -> Result<()> {
let Some(validator) = &self.validator else {
@@ -132,6 +175,18 @@ impl ClassifierContract {
}
}
+fn validate_prompt(prompt_template: &str) -> Result<()> {
+ if prompt_template.trim().is_empty() {
+ return Err(algorithm_error("classifier prompt must not be empty"));
+ }
+ if prompt_template.contains("{{RESPONSE_SCHEMA}}") {
+ return Err(algorithm_error(
+ "classifier prompt must not include {{RESPONSE_SCHEMA}}; remove the placeholder because Switchyard supplies the schema automatically",
+ ));
+ }
+ Ok(())
+}
+
fn compile_schema(schema: &Value) -> Result<Validator> {
if !schema.is_object() {
return Err(algorithm_error("response_schema must be a JSON object"));
@@ -223,7 +278,11 @@ mod tests {
)
.expect_err("schema placeholders should be rejected");
- assert!(error.to_string().contains("schema is sent separately"));
+ assert!(
+ error
+ .to_string()
+ .contains("Switchyard supplies the schema automatically")
+ );
}
#[test]
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 63223bd6c..29d7a49c5 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -62,9 +62,19 @@ where
fn decode(
&self,
response: &AggLlmResponse,
- _contract: &ClassifierContract,
+ contract: &ClassifierContract,
) -> Result<Self::Verdict> {
- parse_json_verdict(response)
+ if !contract.validates_locally() {
+ return parse_json_verdict(response);
+ }
+ let verdict = parse_json_verdict::<Value>(response)?;
+ contract.validate_verdict(&verdict)?;
+ serde_json::from_value(verdict).map_err(|error| LibsyError::AlgorithmError {
+ message: format!(
+ "judge reply did not parse as {}: {error}",
+ std::any::type_name::<Self::Verdict>()
+ ),
+ })
}
}
@@ -350,6 +360,11 @@ mod tests {
ok: bool,
}
+ #[derive(Debug, Deserialize, PartialEq)]
+ struct ScoreVerdict {
+ score: f64,
+ }
+
struct TestJudge;
impl Judge for TestJudge {
@@ -413,6 +428,47 @@ mod tests {
Ok(())
}
+ #[test]
+ fn typed_decoder_enforces_a_json_object_contract_locally() -> Result<()> {
+ use super::super::classifier_contract::{
+ ClassifierContractConfig, ClassifierResponseFormat,
+ };
+
+ let config = ClassifierContractConfig::default()
+ .with_response_format_type(ClassifierResponseFormat::JsonObject);
+ let contract = ClassifierContract::from_config(
+ &config,
+ "Return one JSON score.",
+ r#"{
+ "type": "json_schema",
+ "json_schema": {
+ "name": "ScoreVerdict",
+ "schema": {
+ "type": "object",
+ "properties": {"score": {"type": "number"}},
+ "required": ["score"],
+ "additionalProperties": false
+ }
+ }
+ }"#,
+ )?;
+ let decoder = SerdeDecoder::<ScoreVerdict>::new();
+
+ let error = decoder
+ .decode(
+ &text_response(None, r#"{"score":0.5,"unexpected":true}"#),
+ &contract,
+ )
+ .expect_err("an extra property should fail the local schema");
+
+ assert!(error.to_string().contains("did not match response_schema"));
+ assert_eq!(
+ decoder.decode(&text_response(None, r#"{"score":0.5}"#), &contract)?,
+ ScoreVerdict { score: 0.5 }
+ );
+ Ok(())
+ }
+
fn buffered(completion: &str) -> Response {
Response {
llm_response: LlmResponse::Agg(text_response(None, completion)),
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 3b1ede9ac..583ce1f13 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -23,7 +23,9 @@ pub use algorithms::passthrough::Passthrough;
pub use algorithms::rand::{Random, RandomClassifier};
pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig};
pub use algorithms::util::affinity::AffinityRouter;
-pub use algorithms::util::classifier_contract::ClassifierContractConfig;
+pub use algorithms::util::classifier_contract::{
+ ClassifierContractConfig, ClassifierResponseFormat,
+};
pub use algorithms::util::escalation::EscalationJudgeConfig;
pub use algorithms::util::prompts::{SystemPromptProcessor, TargetPrompts, append_note};
pub use algorithms::util::subagent::SubagentOverride;
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 3b3423bb9..78a095260 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -11,7 +11,7 @@ use http::header::{HeaderName, HeaderValue};
use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError};
use pyo3::prelude::*;
use switchyard_libsy::{
- Algorithm, CallModel, ClassifierContractConfig, HandoffNoteConfig,
+ Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, HandoffNoteConfig,
LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop,
PickerMode, Random, StageRouter, StageRouterConfig, Step as RustStep, StepStream,
TaskClassifierConfig,
@@ -68,7 +68,8 @@ impl PyTaskClassifierConfig {
message_hash_fallback=false,
recent_turn_window=None,
max_output_tokens=4096,
- prompt=None
+ prompt=None,
+ response_format_type="json_schema"
))]
#[allow(clippy::too_many_arguments)]
fn new(
@@ -79,12 +80,23 @@ impl PyTaskClassifierConfig {
recent_turn_window: Option<usize>,
max_output_tokens: u64,
prompt: Option<String>,
- ) -> Self {
+ response_format_type: &str,
+ ) -> PyResult<Self> {
let mut contract = ClassifierContractConfig::default();
if let Some(prompt) = prompt {
contract = contract.with_prompt(prompt);
}
- Self {
+ let response_format_type = match response_format_type {
+ "json_schema" => ClassifierResponseFormat::JsonSchema,
+ "json_object" => ClassifierResponseFormat::JsonObject,
+ other => {
+ return Err(PyValueError::new_err(format!(
+ "response_format_type must be 'json_schema' or 'json_object', got {other:?}"
+ )));
+ }
+ };
+ contract = contract.with_response_format_type(response_format_type);
+ Ok(Self {
inner: TaskClassifierConfig {
base_threshold,
threshold_step,
@@ -94,7 +106,7 @@ impl PyTaskClassifierConfig {
contract,
max_output_tokens,
},
- }
+ })
}
}
diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs
index c69e4374a..561390235 100644
--- a/crates/switchyard-server/src/config.rs
+++ b/crates/switchyard-server/src/config.rs
@@ -9,10 +9,10 @@ use std::path::Path;
use std::sync::Arc;
use libsy::{
- Algorithm, ClassifierContractConfig, CustomClassifierConfig, CustomClassifierPolicy,
- EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier,
- Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts,
- TaskClassifierConfig,
+ Algorithm, ClassifierContractConfig, ClassifierResponseFormat, CustomClassifierConfig,
+ CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig,
+ LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter,
+ StageRouterConfig, TargetPrompts, TaskClassifierConfig,
};
use serde::Deserialize;
use serde_json::Value;
@@ -325,6 +325,7 @@ struct CapabilityClassifierRouteConfig {
message_hash_fallback: bool,
recent_turn_window: Option<usize>,
prompt: Option<String>,
+ response_format_type: ClassifierResponseFormat,
max_output_tokens: u64,
}
@@ -333,6 +334,7 @@ struct EscalationClassifierRouteConfig {
strong_target: String,
weak_target: String,
prompt: Option<String>,
+ response_format_type: ClassifierResponseFormat,
max_output_tokens: u64,
judge: EscalationJudgeConfig,
}
@@ -411,6 +413,8 @@ enum RouteConfig {
recent_turn_window: Option<usize>,
#[serde(default)]
prompt: Option<String>,
+ #[serde(default)]
+ response_format_type: ClassifierResponseFormat,
#[serde(default = "default_classifier_max_output_tokens")]
max_output_tokens: u64,
#[serde(default)]
@@ -471,6 +475,8 @@ struct StageClassifierConfig {
recent_turn_window: Option<usize>,
#[serde(default)]
prompt: Option<String>,
+ #[serde(default)]
+ response_format_type: ClassifierResponseFormat,
#[serde(default = "default_classifier_max_output_tokens")]
max_output_tokens: u64,
}
@@ -483,7 +489,8 @@ impl StageClassifierConfig {
session_affinity: self.session_affinity,
message_hash_fallback: self.message_hash_fallback,
recent_turn_window: self.recent_turn_window,
- contract: classifier_contract(self.prompt.as_deref()),
+ contract: classifier_contract(self.prompt.as_deref())
+ .with_response_format_type(self.response_format_type),
max_output_tokens: self.max_output_tokens,
}
}
@@ -609,6 +616,7 @@ impl RouteConfig {
message_hash_fallback,
recent_turn_window,
prompt,
+ response_format_type,
max_output_tokens,
escalation,
targets,
@@ -666,6 +674,7 @@ impl RouteConfig {
message_hash_fallback: *message_hash_fallback,
recent_turn_window: *recent_turn_window,
prompt: prompt.clone(),
+ response_format_type: *response_format_type,
max_output_tokens: *max_output_tokens,
},
))
@@ -703,6 +712,7 @@ impl RouteConfig {
weak_target,
)?,
prompt: prompt.clone(),
+ response_format_type: *response_format_type,
max_output_tokens: *max_output_tokens,
judge: required_classifier_field(route_name, "escalation", escalation)?,
},
@@ -714,6 +724,7 @@ impl RouteConfig {
|| base_threshold.is_some()
|| threshold_step.is_some()
|| escalation.is_some()
+ || *response_format_type != ClassifierResponseFormat::JsonSchema
{
return Err(ServerError::new(format!(
"llm_classifier route {route_name} mode custom cannot use capability or escalation fields"
@@ -884,7 +895,8 @@ fn build_algorithm(
session_affinity: config.session_affinity,
message_hash_fallback: config.message_hash_fallback,
recent_turn_window: config.recent_turn_window,
- contract: classifier_contract(config.prompt.as_deref()),
+ contract: classifier_contract(config.prompt.as_deref())
+ .with_response_format_type(config.response_format_type),
max_output_tokens: config.max_output_tokens,
};
LlmTaskClassifier::new(LlmClassifierConfig::Capability {
@@ -902,7 +914,8 @@ fn build_algorithm(
judge_target: classifier,
efficient_target: weak,
capable_target: strong,
- contract: classifier_contract(config.prompt.as_deref()),
+ contract: classifier_contract(config.prompt.as_deref())
+ .with_response_format_type(config.response_format_type),
config: config.judge,
max_output_tokens: config.max_output_tokens,
})
@@ -1192,7 +1205,10 @@ target = "weak"
"base_threshold = 0.5",
"base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"",
);
- assert!(error_message(&schema_placeholder).contains("schema is sent separately"));
+ assert!(
+ error_message(&schema_placeholder)
+ .contains("Switchyard supplies the schema automatically")
+ );
Ok(())
}
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index a0e38159b..868b9a858 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -173,6 +173,13 @@ async fn upstream_chat(
.is_some_and(|content| content.contains("invalid verdict"))
})
});
+ let requests_schema_invalid_verdict = body["messages"].as_array().is_some_and(|messages| {
+ messages.iter().any(|message| {
+ message["content"]
+ .as_str()
+ .is_some_and(|content| content.contains("schema-invalid verdict"))
+ })
+ });
let content = if model == "model/classifier" && custom_target_schema {
if requests_invalid_verdict {
r#"{"decision":{"target":"unknown"}}"#
@@ -185,6 +192,8 @@ async fn upstream_chat(
.is_some()
{
r#"{"escalate":false,"reason":"making progress"}"#
+ } else if model == "model/classifier" && requests_schema_invalid_verdict {
+ r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.1,"unexpected":true}"#
} else if model == "model/classifier" {
r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#
} else {
@@ -1152,7 +1161,7 @@ selector = "/decision/target"
}
#[tokio::test]
-async fn classifier_prompt_overrides_reach_every_server_mode() -> TestResult {
+async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult {
let upstream = MockUpstream::start().await?;
let state = load_test_config(&format!(
r#"
@@ -1183,6 +1192,7 @@ strong_target = "strong"
weak_target = "weak"
base_threshold = 0.5
prompt = "CUSTOM CAPABILITY"
+response_format_type = "json_object"
[routes.escalation]
id = "switchyard/escalation"
@@ -1192,6 +1202,7 @@ classifier_target = "classifier"
strong_target = "strong"
weak_target = "weak"
prompt = "CUSTOM ESCALATION"
+response_format_type = "json_object"
escalation = {{ confirmations = 1 }}
[routes.stage]
@@ -1211,10 +1222,20 @@ prompt = "CUSTOM STAGE"
))?;
let app = build_switchyard_router(state);
- for (route, prompt_prefix, schema_field) in [
- ("switchyard/capability", "CUSTOM CAPABILITY", "p_solve"),
- ("switchyard/escalation", "CUSTOM ESCALATION", "escalate"),
- ("switchyard/stage", "CUSTOM STAGE", "p_solve"),
+ for (route, prompt_prefix, schema_field, json_object) in [
+ (
+ "switchyard/capability",
+ "CUSTOM CAPABILITY",
+ "p_solve",
+ true,
+ ),
+ (
+ "switchyard/escalation",
+ "CUSTOM ESCALATION",
+ "escalate",
+ true,
+ ),
+ ("switchyard/stage", "CUSTOM STAGE", "p_solve", false),
] {
upstream.calls.lock().await.clear();
let response = send(
@@ -1238,16 +1259,118 @@ prompt = "CUSTOM STAGE"
.as_str()
.ok_or("classifier prompt was not text")?;
assert!(prompt.starts_with(prompt_prefix), "{route}: {prompt}");
- assert!(
- judge_call["response_format"]["json_schema"]["schema"]["properties"]
- .get(schema_field)
- .is_some(),
- "{route}: missing {schema_field} in {judge_call}"
- );
+ if json_object {
+ assert_eq!(
+ judge_call["response_format"],
+ json!({"type": "json_object"}),
+ "{route}: {judge_call}"
+ );
+ assert!(prompt.contains("JSON Schema"), "{route}: {prompt}");
+ assert!(
+ prompt.contains(&format!("\"{schema_field}\"")),
+ "{route}: missing {schema_field} in {prompt}"
+ );
+ } else {
+ assert!(
+ judge_call["response_format"]["json_schema"]["schema"]["properties"]
+ .get(schema_field)
+ .is_some(),
+ "{route}: missing {schema_field} in {judge_call}"
+ );
+ }
}
Ok(())
}
+#[tokio::test]
+async fn stage_classifier_can_request_json_object_output() -> TestResult {
+ let upstream = MockUpstream::start().await?;
+ let state = load_test_config(&format!(
+ r#"
+schema_version = 1
+
+[llm_clients.upstream]
+format = "openai_chat"
+base_url = "{base_url}"
+
+[targets.classifier]
+id = "model/classifier"
+llm_client = "upstream"
+
+[targets.strong]
+id = "model/strong"
+llm_client = "upstream"
+
+[targets.weak]
+id = "model/weak"
+llm_client = "upstream"
+
+[routes.stage]
+id = "switchyard/stage"
+type = "stage_router"
+capable_target = "strong"
+efficient_target = "weak"
+picker = "efficient_first"
+confidence_threshold = 1.0
+
+[routes.stage.classifier]
+target = "classifier"
+base_threshold = 0.5
+response_format_type = "json_object"
+"#,
+ base_url = upstream.base_url
+ ))?;
+ let app = build_switchyard_router(state);
+
+ let response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/stage",
+ "messages": [{"role": "user", "content": "bounded task"}]
+ })),
+ )
+ .await?;
+
+ assert_eq!(response.status, StatusCode::OK);
+ let calls = upstream.calls.lock().await;
+ let judge_call = calls
+ .iter()
+ .find(|call| call["model"] == "model/classifier")
+ .ok_or("classifier target was not called")?;
+ assert_eq!(
+ judge_call["response_format"],
+ json!({"type": "json_object"})
+ );
+ let prompt = judge_call["messages"][0]["content"]
+ .as_str()
+ .ok_or("classifier prompt was not text")?;
+ assert!(prompt.contains("JSON Schema"), "{prompt}");
+ assert!(prompt.contains("\"p_solve\""), "{prompt}");
+
+ drop(calls);
+ let invalid_response = send(
+ &app,
+ "POST",
+ "/v1/chat/completions",
+ Some(json!({
+ "model": "switchyard/stage",
+ "messages": [{"role": "user", "content": "return a schema-invalid verdict"}]
+ })),
+ )
+ .await?;
+ assert_eq!(invalid_response.status, StatusCode::OK);
+ assert_eq!(
+ invalid_response
+ .headers
+ .get("x-model-router-selected-model")
+ .and_then(|value| value.to_str().ok()),
+ Some("model/weak")
+ );
+ Ok(())
+}
+
#[tokio::test]
async fn count_tokens_forwards_to_configured_anthropic_target() -> TestResult {
let upstream = MockUpstream::start().await?;
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 6b7a8f9ef..68d01bc54 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -143,6 +143,7 @@ Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`.
| `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. |
| `classifier_target` | Yes | — | Target the judge is called through. Not a routing destination. |
| `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. |
+| `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. |
Capability mode classifies before serving. See
[LLM Classifier Routing](../routing_algorithms/llm_classifier_routing.md).
@@ -186,8 +187,9 @@ policy selector, and routes to any configured target label.
| `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `session_affinity`. |
| `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. |
-Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard sends the
-schema only through the provider's structured-output request.
+Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies
+the schema automatically: through the structured-output request in `json_schema`
+mode, or in the prompt in `json_object` mode.
### `stage_router`
@@ -204,6 +206,7 @@ optional `handoff_notes` and `classifier` tables and for tuning.
| `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. |
| `capable_system_prompt` | No | unset | System prompt handed to the capable tier. |
| `efficient_system_prompt` | No | unset | System prompt handed to the efficient tier. |
+| `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. |
## Validation Errors
diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md
index e2246144e..1ed9e8085 100644
--- a/docs/routing_algorithms/escalation_router_routing.md
+++ b/docs/routing_algorithms/escalation_router_routing.md
@@ -48,8 +48,9 @@ clients send; the judge is not exposed as a client-selectable model.
The route-level `prompt` key replaces the packaged trajectory-judge prompt. It
uses the escalation verdict schema rather than the capability verdict schema.
-Switchyard sends that schema separately through the provider's structured-output
-request rather than copying it into the prompt.
+Switchyard supplies that schema according to the route's `response_format_type`:
+through the structured-output request in the default `json_schema` mode, or in
+the prompt in `json_object` mode.
## How the decision works
diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md
index e3f750974..a1cc43e45 100644
--- a/docs/routing_algorithms/llm_classifier_routing.md
+++ b/docs/routing_algorithms/llm_classifier_routing.md
@@ -78,6 +78,11 @@ Switchyard does not parse provider-specific reasoning fields such as
to `strong_target` even when the judge request returned HTTP 200. With session
affinity, that fallback can be reused without another judge call.
+Capability and escalation routes use JSON Schema structured output by default.
+For a provider that supports JSON Object mode but not JSON Schema, set
+`response_format_type = "json_object"` on the route. Switchyard then adds the
+verdict schema to the judge prompt and validates the returned object locally.
+
When a vLLM-compatible provider supports `enable_thinking`, configure it on the
judge target through `extra_body`:
@@ -104,13 +109,14 @@ for the server merge behavior.
| `session_affinity` | `false` | Retains the first selected target for a session and reuses it on later requests. |
| `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `session_affinity = true`. |
| `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. |
+| `response_format_type` | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` for providers without JSON Schema support. |
| `max_output_tokens` | `4096` | Maximum completion tokens available to the classifier verdict. Must be at least `1`. |
### Override the classifier prompt
Set `prompt` on the route when the packaged capability rubric does not describe
-your weak model. Switchyard sends the response schema separately through the
-provider's structured-output request; do not copy it into the prompt.
+your weak model. Do not copy the response schema into the prompt: Switchyard
+supplies it according to `response_format_type`.
```toml
[routes.smart]
diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md
index c471a9882..7bf0da7bb 100644
--- a/docs/routing_algorithms/stage_router_routing.md
+++ b/docs/routing_algorithms/stage_router_routing.md
@@ -245,11 +245,14 @@ base_threshold = 0.5 # p_solve floor to route efficient; below this → ca
threshold_step = 0.1 # adds 0.1 for uncertain and 0.2 for unsupported verdicts
recent_turn_window = 3 # conversation span the judge sees
prompt = "Estimate whether the efficient target can complete this request."
+response_format_type = "json_object" # optional; default is "json_schema"
```
-`prompt` replaces the packaged capability-classifier prompt. The active schema
-is sent separately through the structured-output request. The verdict schema
-and routing thresholds remain unchanged.
+`prompt` replaces the packaged capability-classifier prompt. In the default
+`json_schema` mode, Switchyard sends the verdict schema through the structured-output
+request. Set `response_format_type = "json_object"` for providers that only support
+JSON Object mode; Switchyard then adds the schema to the judge prompt and validates
+the returned object locally. The verdict schema and routing thresholds remain unchanged.
Give the classifier its own LLM client or quota bucket where possible. Sharing
one provider bucket with the efficient tier adds a request per classified turn
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index 5dd607566..498eeca05 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -95,6 +95,7 @@ if TYPE_CHECKING:
recent_turn_window: int | None = None,
max_output_tokens: int = 4096,
prompt: str | None = None,
+ response_format_type: Literal["json_schema", "json_object"] = "json_schema",
) -> None: ...
@final
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index 00ca23b1c..15b7b5843 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -181,6 +181,66 @@ async def test_classifier_config_accepts_a_prompt_override() -> None:
assert response["model"] == "weak"
+async def test_classifier_config_accepts_json_object_output() -> None:
+ """Verify that Python can select JSON Object mode for a classifier judge."""
+
+ class JudgeClient(EchoClient):
+ async def call(self, request: dict[str, Any]) -> dict[str, Any]:
+ self.calls.append(request)
+ return {
+ "model": self.model,
+ "outputs": [
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ '{"crux":"bounded task","primary_rule":"SUP-1",'
+ '"capability_boundary":"supported","p_solve":0.9}'
+ ),
+ }
+ ],
+ "stop_reason": "end_turn",
+ }
+ ],
+ }
+
+ judge = JudgeClient("judge")
+ weak = EchoClient("weak")
+ algorithm = algorithms.llm_task_classifier(
+ "judge",
+ "weak",
+ "strong",
+ config=TaskClassifierConfig(0.5, response_format_type="json_object"),
+ )
+
+ _, response = await run_algorithm(
+ algorithm,
+ {
+ "judge": judge,
+ "weak": weak,
+ "strong": EchoClient("strong"),
+ },
+ )
+
+ assert judge.calls[0]["output"]["response_format"] == {"type": "json_object"}
+ prompt = judge.calls[0]["instructions"][0]["content"][0]["text"]
+ assert "JSON Schema" in prompt
+ assert '"p_solve"' in prompt
+ assert response["model"] == "weak"
+
+
+def test_classifier_config_rejects_unknown_response_format() -> None:
+ invalid_response_format: Any = "yaml"
+
+ with pytest.raises(
+ ValueError,
+ match="response_format_type must be 'json_schema' or 'json_object'",
+ ):
+ TaskClassifierConfig(0.5, response_format_type=invalid_response_format)
+
+
async def test_random_weights_and_seed_are_reproducible() -> None:
def algorithm():
return algorithms.random(