Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
fix(decision): Correct `fail_open` handling. (#1198)
Graham King839017a2166d
diff --git a/bindings/typescript/test/composite.test.mjs b/bindings/typescript/test/composite.test.mjsindex 062e7fc3a..c7b5184a1 100644--- a/bindings/typescript/test/composite.test.mjs+++ b/bindings/typescript/test/composite.test.mjs@@ -159,7 +159,20 @@ test("validation and decision failure policy", async () => {model: options.targets.efficient,state: { stage_previous_tier: { string: "weak" } },});- assert.deepEqual(await router.route(messages, { state, decide: async () => ({ answers: {} }) }), { model: options.targets.capable });+ for (const answer of [+ { answers: {} },+ { answers: { route: { value: { type: "boolean", data: { type: "value", data: true } } } } },+ { answers: { route: { value: { type: "choice", data: { selected: "advantage" } } } } },+ { answers: { route: { value: { type: "choice", data: { selected: "no_advantage", probabilities: { no_advantage: 1 } } } } } },+ ]) {+ const decide = async () => answer;+ await assert.rejects(router.route(messages, { state, decide }), { code: "ERR_ROUTING" });+ assert.deepEqual(await fallback.route(messages, { state, decide }), { model: options.targets.capable });+ assert.deepEqual(await fallback.route(messages, { decide }), {+ model: options.targets.efficient,+ state: { stage_previous_tier: { string: "weak" } },+ });+ }});test("abort settles promptly, ignores late answers, and removes listeners", { timeout: 5000 }, async () => {diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rsindex 9e45aef00..26a916343 100644--- a/crates/libsy-llm-client/src/observability.rs+++ b/crates/libsy-llm-client/src/observability.rs@@ -200,6 +200,7 @@ fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),LibsyError::NoTargets => Cow::Borrowed("no_targets"),LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),+ LibsyError::InvalidDecision { .. } => Cow::Borrowed("invalid_decision"),LibsyError::Driver(_) => Cow::Borrowed("driver_error"),LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),LibsyError::External { .. } => Cow::Borrowed("_OTHER"),diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 66f2288cf..d9921d244 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -1578,7 +1578,16 @@ mod tests {("no distribution", 0.0, true, Some("capable")),("wrong type", 0.0, true, Some("capable")),("out of range", 1.1, true, Some("capable")),- ("nonfinite", f64::NAN, false, Some("capable")),+ ("nonfinite", f64::NAN, true, Some("capable")),+ ("missing probability", 0.0, true, Some("capable")),+ ("missing answer", 0.0, false, None),+ ("no distribution", 0.0, false, None),+ ("wrong type", 0.0, false, None),+ ("out of range", 1.1, false, None),+ ("negative", -0.1, false, None),+ ("nonfinite", f64::NAN, false, None),+ ("infinite", f64::INFINITY, false, None),+ ("missing probability", 0.0, false, None),("provider error", 0.0, true, Some("capable")),("provider error", 0.0, false, None),("dropped reply", 0.0, true, Some("capable")),@@ -1666,7 +1675,15 @@ mod tests {selected: "no_advantage".into(),probabilities: (name != "no distribution").then(|| {BTreeMap::from([- ("advantage".into(), Probability(score)),+ (+ if name == "missing probability" {+ "other"+ } else {+ "advantage"+ }+ .into(),+ Probability(score),+ ),("no_advantage".into(), Probability(1.0 - score)),])}),@@ -1704,11 +1721,16 @@ mod tests {assert!(matches!(result, Err(LibsyError::AlgorithmError { message }) if message.contains("candidate is missing")));- } else {+ } else if name == "provider error" {assert!(matches!(result, Err(LibsyError::ClientCall { .. })),"{name}");+ } else {+ assert!(+ matches!(result, Err(LibsyError::InvalidDecision { target, .. }) if target == "judge"),+ "expected invalid decision error: {name}"+ );}continue;};diff --git a/crates/libsy/src/algorithms/llm_class/decision.rs b/crates/libsy/src/algorithms/llm_class/decision.rsindex bf67f1fa1..9142107fd 100644--- a/crates/libsy/src/algorithms/llm_class/decision.rs+++ b/crates/libsy/src/algorithms/llm_class/decision.rs@@ -127,6 +127,48 @@ impl DecisionClassifier {question,})}++ async fn try_score(+ &self,+ decision: DecisionRequest,+ driver: &Driver,+ judge: &ModelId,+ score_option: &str,+ ) -> Result<f64> {+ let response = driver.call_decision(decision, judge.clone()).await?;+ let answer = response+ .answers+ .get("route")+ .ok_or_else(|| LibsyError::invalid_decision(judge.clone(), "missing route answer"))?;+ let DecisionValue::Choice {+ probabilities: Some(probabilities),+ ..+ } = &answer.value+ else {+ return Err(LibsyError::invalid_decision(+ judge.clone(),+ "expected a choice with probabilities",+ ));+ };+ // The selected option can differ from the cutoff event, so only the+ // requested option's probability is used.+ let score = probabilities+ .get(score_option)+ .map(|p| p.0)+ .ok_or_else(|| {+ LibsyError::invalid_decision(+ judge.clone(),+ format!("missing probability for {score_option}"),+ )+ })?;+ if !(0.0..=1.0).contains(&score) {+ return Err(LibsyError::invalid_decision(+ judge.clone(),+ "decision probability must be between 0 and 1",+ ));+ }+ Ok(score)+ }}#[async_trait]@@ -159,8 +201,8 @@ impl Classifier for DecisionClassifier {}),questions: BTreeMap::from([("route".into(), self.question.clone())]),};- let response = match driver.call_decision(decision, judge.clone()).await {- Ok(response) => response,+ let score = match self.try_score(decision, driver, judge, score_option).await {+ Ok(score) => score,Err(error) if self.fail_open => {return Ok(unavailable(driver,@@ -171,27 +213,6 @@ impl Classifier for DecisionClassifier {}Err(error) => return Err(error),};- // The provider's selected option can differ from the application's cutoff.- // Only the requested event's score is used; confidence is a separate signal.- let score = response- .answers- .get("route")- .and_then(|answer| match &answer.value {- DecisionValue::Choice {- probabilities: Some(probabilities),- ..- } => probabilities.get(score_option).map(|p| p.0),- _ => None,- })- .filter(|p| (0.0..=1.0).contains(p));- let Some(score) = score else {- return Ok(unavailable(- driver,- judge,- format!("missing or invalid {score_option} score"),- "invalid_verdict",- ));- };let (target, category) = if score > self.config.cutoff {(capable, Category::Capable)} else {diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex a416623e8..230db1856 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -411,6 +411,7 @@ pub(crate) fn report_fail_open(judge_model: &str, error: String, reason: &'statipub(crate) fn libsy_error_reason(error: &LibsyError) -> &'static str {match error {LibsyError::ClientCall { source, .. } => client_error_reason(source),+ LibsyError::InvalidDecision { .. } => "invalid_verdict",_ => "call_error",}}diff --git a/crates/libsy/src/algorithms/util/robustness.rs b/crates/libsy/src/algorithms/util/robustness.rsindex 20d610195..9f13e8413 100644--- a/crates/libsy/src/algorithms/util/robustness.rs+++ b/crates/libsy/src/algorithms/util/robustness.rs@@ -33,6 +33,7 @@ pub(crate) fn safe_error_summary(error: &LibsyError) -> String {// from static text today, but nothing stops one embedding a response or an// upstream error, so only the class is reported.LibsyError::AlgorithmError { .. } => "algorithm error".to_string(),+ LibsyError::InvalidDecision { .. } => "invalid decision answer".to_string(),LibsyError::Driver(_) => "algorithm driver failed".to_string(),// The operation is a static label, but the boxed source comes from a user// extension and is unvetted.@@ -84,6 +85,13 @@ mod tests {/// The conversation text a leaking log line would expose.const SECRET: &str = "patient name is Jane Doe";+ #[test]+ fn invalid_decision_details_are_redacted() {+ let error = LibsyError::invalid_decision("judge", SECRET);+ assert_eq!(safe_error_summary(&error), "invalid decision answer");+ assert!(error.to_string().contains(SECRET));+ }+#[test]fn an_upstream_body_never_reaches_the_summary() {let error = LibsyError::ClientCall {diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rsindex 40eceb829..c7ac5c6b9 100644--- a/crates/libsy/src/error.rs+++ b/crates/libsy/src/error.rs@@ -50,6 +50,15 @@ pub enum LibsyError {source: LlmClientError,},+ /// A decision judge returned an unusable answer.+ #[error("invalid decision from target {target:?}: {message}")]+ InvalidDecision {+ /// Judge that supplied the answer.+ target: ModelId,+ /// Description of the answer validation failure.+ message: String,+ },+/// A user extension or other foreign operation failed.#[error("{operation} failed: {source}")]External {@@ -70,6 +79,14 @@ impl LibsyError {}}+ /// Describe an invalid answer returned by a decision judge.+ pub fn invalid_decision(target: impl Into<ModelId>, message: impl Into<String>) -> Self {+ Self::InvalidDecision {+ target: target.into(),+ message: message.into(),+ }+ }+/// Preserve a concrete foreign error with a description of the failed operation.pub fn external(operation: &'static str,diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.mdindex 7e086c879..eac1c1eaa 100644--- a/docs/reference/toml_schema.md+++ b/docs/reference/toml_schema.md@@ -353,7 +353,7 @@ Capability mode classifies before serving. See| `strong_target` | Yes | — | Capable tier. || `weak_target` | Yes | — | Efficient tier. || `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. |-| `fail_open` | No | `true` | Capability mode only. When `true`, judge client failures and deadlines route to the capable tier and record fail-open evidence. When `false`, they stop the request. Invalid verdicts use the capable tier with either setting. |+| `fail_open` | No | `true` | Capability mode only. When `true`, judge client failures and deadlines route to the capable tier and record fail-open evidence. When `false`, they stop the request. Invalid decision-model verdicts route to the capable tier when `fail_open = true` and stop the request with an error when `fail_open = false`. Invalid LLM-judge verdicts still use the capable tier with either setting. || `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. || `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. || `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. |