Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(decision): Correct `fail_open` handling. (#1198)

Graham King839017a2166d
diff --git a/bindings/typescript/test/composite.test.mjs b/bindings/typescript/test/composite.test.mjs
index 062e7fc3a..c7b5184a1 100644
--- a/bindings/typescript/test/composite.test.mjs
+++ b/bindings/typescript/test/composite.test.mjs
@@ -159,7 +159,20 @@ test("validation and decision failure policy", async () => {
model: options.targets.efficient,
state: { stage_previous_tier: { string: "weak" } },
});
- assert.deepEqual(await router.route(messages, { state, decide: async () => ({ answers: {} }) }), { model: options.targets.capable });
+ for (const answer of [
+ { answers: {} },
+ { answers: { route: { value: { type: "boolean", data: { type: "value", data: true } } } } },
+ { answers: { route: { value: { type: "choice", data: { selected: "advantage" } } } } },
+ { answers: { route: { value: { type: "choice", data: { selected: "no_advantage", probabilities: { no_advantage: 1 } } } } } },
+ ]) {
+ const decide = async () => answer;
+ await assert.rejects(router.route(messages, { state, decide }), { code: "ERR_ROUTING" });
+ assert.deepEqual(await fallback.route(messages, { state, decide }), { model: options.targets.capable });
+ assert.deepEqual(await fallback.route(messages, { decide }), {
+ model: options.targets.efficient,
+ state: { stage_previous_tier: { string: "weak" } },
+ });
+ }
});
test("abort settles promptly, ignores late answers, and removes listeners", { timeout: 5000 }, async () => {
diff --git a/crates/libsy-llm-client/src/observability.rs b/crates/libsy-llm-client/src/observability.rs
index 9e45aef00..26a916343 100644
--- a/crates/libsy-llm-client/src/observability.rs
+++ b/crates/libsy-llm-client/src/observability.rs
@@ -200,6 +200,7 @@ fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {
LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),
LibsyError::NoTargets => Cow::Borrowed("no_targets"),
LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),
+ LibsyError::InvalidDecision { .. } => Cow::Borrowed("invalid_decision"),
LibsyError::Driver(_) => Cow::Borrowed("driver_error"),
LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),
LibsyError::External { .. } => Cow::Borrowed("_OTHER"),
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 66f2288cf..d9921d244 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -1578,7 +1578,16 @@ mod tests {
("no distribution", 0.0, true, Some("capable")),
("wrong type", 0.0, true, Some("capable")),
("out of range", 1.1, true, Some("capable")),
- ("nonfinite", f64::NAN, false, Some("capable")),
+ ("nonfinite", f64::NAN, true, Some("capable")),
+ ("missing probability", 0.0, true, Some("capable")),
+ ("missing answer", 0.0, false, None),
+ ("no distribution", 0.0, false, None),
+ ("wrong type", 0.0, false, None),
+ ("out of range", 1.1, false, None),
+ ("negative", -0.1, false, None),
+ ("nonfinite", f64::NAN, false, None),
+ ("infinite", f64::INFINITY, false, None),
+ ("missing probability", 0.0, false, None),
("provider error", 0.0, true, Some("capable")),
("provider error", 0.0, false, None),
("dropped reply", 0.0, true, Some("capable")),
@@ -1666,7 +1675,15 @@ mod tests {
selected: "no_advantage".into(),
probabilities: (name != "no distribution").then(|| {
BTreeMap::from([
- ("advantage".into(), Probability(score)),
+ (
+ if name == "missing probability" {
+ "other"
+ } else {
+ "advantage"
+ }
+ .into(),
+ Probability(score),
+ ),
("no_advantage".into(), Probability(1.0 - score)),
])
}),
@@ -1704,11 +1721,16 @@ mod tests {
assert!(
matches!(result, Err(LibsyError::AlgorithmError { message }) if message.contains("candidate is missing"))
);
- } else {
+ } else if name == "provider error" {
assert!(
matches!(result, Err(LibsyError::ClientCall { .. })),
"{name}"
);
+ } else {
+ assert!(
+ matches!(result, Err(LibsyError::InvalidDecision { target, .. }) if target == "judge"),
+ "expected invalid decision error: {name}"
+ );
}
continue;
};
diff --git a/crates/libsy/src/algorithms/llm_class/decision.rs b/crates/libsy/src/algorithms/llm_class/decision.rs
index bf67f1fa1..9142107fd 100644
--- a/crates/libsy/src/algorithms/llm_class/decision.rs
+++ b/crates/libsy/src/algorithms/llm_class/decision.rs
@@ -127,6 +127,48 @@ impl DecisionClassifier {
question,
})
}
+
+ async fn try_score(
+ &self,
+ decision: DecisionRequest,
+ driver: &Driver,
+ judge: &ModelId,
+ score_option: &str,
+ ) -> Result<f64> {
+ let response = driver.call_decision(decision, judge.clone()).await?;
+ let answer = response
+ .answers
+ .get("route")
+ .ok_or_else(|| LibsyError::invalid_decision(judge.clone(), "missing route answer"))?;
+ let DecisionValue::Choice {
+ probabilities: Some(probabilities),
+ ..
+ } = &answer.value
+ else {
+ return Err(LibsyError::invalid_decision(
+ judge.clone(),
+ "expected a choice with probabilities",
+ ));
+ };
+ // The selected option can differ from the cutoff event, so only the
+ // requested option's probability is used.
+ let score = probabilities
+ .get(score_option)
+ .map(|p| p.0)
+ .ok_or_else(|| {
+ LibsyError::invalid_decision(
+ judge.clone(),
+ format!("missing probability for {score_option}"),
+ )
+ })?;
+ if !(0.0..=1.0).contains(&score) {
+ return Err(LibsyError::invalid_decision(
+ judge.clone(),
+ "decision probability must be between 0 and 1",
+ ));
+ }
+ Ok(score)
+ }
}
#[async_trait]
@@ -159,8 +201,8 @@ impl Classifier for DecisionClassifier {
}),
questions: BTreeMap::from([("route".into(), self.question.clone())]),
};
- let response = match driver.call_decision(decision, judge.clone()).await {
- Ok(response) => response,
+ let score = match self.try_score(decision, driver, judge, score_option).await {
+ Ok(score) => score,
Err(error) if self.fail_open => {
return Ok(unavailable(
driver,
@@ -171,27 +213,6 @@ impl Classifier for DecisionClassifier {
}
Err(error) => return Err(error),
};
- // The provider's selected option can differ from the application's cutoff.
- // Only the requested event's score is used; confidence is a separate signal.
- let score = response
- .answers
- .get("route")
- .and_then(|answer| match &answer.value {
- DecisionValue::Choice {
- probabilities: Some(probabilities),
- ..
- } => probabilities.get(score_option).map(|p| p.0),
- _ => None,
- })
- .filter(|p| (0.0..=1.0).contains(p));
- let Some(score) = score else {
- return Ok(unavailable(
- driver,
- judge,
- format!("missing or invalid {score_option} score"),
- "invalid_verdict",
- ));
- };
let (target, category) = if score > self.config.cutoff {
(capable, Category::Capable)
} else {
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index a416623e8..230db1856 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -411,6 +411,7 @@ pub(crate) fn report_fail_open(judge_model: &str, error: String, reason: &'stati
pub(crate) fn libsy_error_reason(error: &LibsyError) -> &'static str {
match error {
LibsyError::ClientCall { source, .. } => client_error_reason(source),
+ LibsyError::InvalidDecision { .. } => "invalid_verdict",
_ => "call_error",
}
}
diff --git a/crates/libsy/src/algorithms/util/robustness.rs b/crates/libsy/src/algorithms/util/robustness.rs
index 20d610195..9f13e8413 100644
--- a/crates/libsy/src/algorithms/util/robustness.rs
+++ b/crates/libsy/src/algorithms/util/robustness.rs
@@ -33,6 +33,7 @@ pub(crate) fn safe_error_summary(error: &LibsyError) -> String {
// from static text today, but nothing stops one embedding a response or an
// upstream error, so only the class is reported.
LibsyError::AlgorithmError { .. } => "algorithm error".to_string(),
+ LibsyError::InvalidDecision { .. } => "invalid decision answer".to_string(),
LibsyError::Driver(_) => "algorithm driver failed".to_string(),
// The operation is a static label, but the boxed source comes from a user
// extension and is unvetted.
@@ -84,6 +85,13 @@ mod tests {
/// The conversation text a leaking log line would expose.
const SECRET: &str = "patient name is Jane Doe";
+ #[test]
+ fn invalid_decision_details_are_redacted() {
+ let error = LibsyError::invalid_decision("judge", SECRET);
+ assert_eq!(safe_error_summary(&error), "invalid decision answer");
+ assert!(error.to_string().contains(SECRET));
+ }
+
#[test]
fn an_upstream_body_never_reaches_the_summary() {
let error = LibsyError::ClientCall {
diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs
index 40eceb829..c7ac5c6b9 100644
--- a/crates/libsy/src/error.rs
+++ b/crates/libsy/src/error.rs
@@ -50,6 +50,15 @@ pub enum LibsyError {
source: LlmClientError,
},
+ /// A decision judge returned an unusable answer.
+ #[error("invalid decision from target {target:?}: {message}")]
+ InvalidDecision {
+ /// Judge that supplied the answer.
+ target: ModelId,
+ /// Description of the answer validation failure.
+ message: String,
+ },
+
/// A user extension or other foreign operation failed.
#[error("{operation} failed: {source}")]
External {
@@ -70,6 +79,14 @@ impl LibsyError {
}
}
+ /// Describe an invalid answer returned by a decision judge.
+ pub fn invalid_decision(target: impl Into<ModelId>, message: impl Into<String>) -> Self {
+ Self::InvalidDecision {
+ target: target.into(),
+ message: message.into(),
+ }
+ }
+
/// Preserve a concrete foreign error with a description of the failed operation.
pub fn external(
operation: &'static str,
diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md
index 7e086c879..eac1c1eaa 100644
--- a/docs/reference/toml_schema.md
+++ b/docs/reference/toml_schema.md
@@ -353,7 +353,7 @@ Capability mode classifies before serving. See
| `strong_target` | Yes | — | Capable tier. |
| `weak_target` | Yes | — | Efficient tier. |
| `base_threshold` | Yes | — | Lowest solve probability that routes to the weak target. In `[0, 1]`. |
-| `fail_open` | No | `true` | Capability mode only. When `true`, judge client failures and deadlines route to the capable tier and record fail-open evidence. When `false`, they stop the request. Invalid verdicts use the capable tier with either setting. |
+| `fail_open` | No | `true` | Capability mode only. When `true`, judge client failures and deadlines route to the capable tier and record fail-open evidence. When `false`, they stop the request. Invalid decision-model verdicts route to the capable tier when `fail_open = true` and stop the request with an error when `fail_open = false`. Invalid LLM-judge verdicts still use the capable tier with either setting. |
| `threshold_step` | No | `0.0` | Finite, non-negative amount added once for uncertain verdicts, `unmatched_steps` times for unmatched verdicts, and twice for unsupported verdicts. `base_threshold + 2 * threshold_step` must be at most `1`. |
| `unmatched_steps` | No | `1` | Threshold steps applied to an unmatched verdict (no capability rule applies). `0`, `1`, or `2`; `2` treats it like an unsupported verdict. |
| `classify_trigger` | No | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and retains that target across intervening tool calls only when requests carry a session ID; without a session ID, it behaves like `every_request`. `new_session` judges once and reuses that target for the session. |