Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): populate bounded outcome evidence (#655)

nachiketb-nvidiaf4152d076fd4
diff --git a/crates/libsy/src/algorithms/advisor_gate.rs b/crates/libsy/src/algorithms/advisor_gate.rs
index 87c15bd1f..72fc2a230 100644
--- a/crates/libsy/src/algorithms/advisor_gate.rs
+++ b/crates/libsy/src/algorithms/advisor_gate.rs
@@ -270,14 +270,34 @@ impl AdvisorGate {
.consult(driver, &request, review_tail.as_deref(), trigger_label)
.await
{
- Ok(ConsultOutcome::Approve) => Ok(RoutingOutcome::answered(
- self.executor.clone(),
- request,
- turn.into_response(),
- )),
- Ok(ConsultOutcome::Redo { plan }) => Ok(self.redo(request, turn, &plan)),
- Ok(ConsultOutcome::Failed) => {
+ Ok(ConsultOutcome::Approve) => {
+ driver.set_evidence(serde_json::json!({
+ "source": "advisor",
+ "verdict": "approve",
+ "trigger": trigger_label,
+ }));
+ Ok(RoutingOutcome::answered(
+ self.executor.clone(),
+ request,
+ turn.into_response(),
+ ))
+ }
+ Ok(ConsultOutcome::Redo { plan }) => {
+ driver.set_evidence(serde_json::json!({
+ "source": "advisor",
+ "verdict": "redo",
+ "trigger": trigger_label,
+ }));
+ Ok(self.redo(request, turn, &plan))
+ }
+ Ok(ConsultOutcome::Failed { reason }) => {
self.budget.refund_failure(scope);
+ driver.set_evidence(serde_json::json!({
+ "source": "advisor",
+ "verdict": "fail_open",
+ "trigger": trigger_label,
+ "reason_code": reason,
+ }));
Ok(RoutingOutcome::answered(
self.executor.clone(),
request,
@@ -361,9 +381,8 @@ impl AdvisorGate {
let agg = match reply {
Ok(agg) => agg,
Err(error) => {
- record_consult_failure(crate::algorithms::util::llm_judge::libsy_error_reason(
- &error,
- ));
+ let reason = crate::algorithms::util::llm_judge::libsy_error_reason(&error);
+ record_consult_failure(reason);
if !self.config.fail_open {
// Surface as an algorithm failure (5xx), never as the
// advisor's own client error: a typed ContextWindowExceeded
@@ -386,7 +405,7 @@ impl AdvisorGate {
reply_head: None,
usage: None,
});
- return Ok(ConsultOutcome::Failed);
+ return Ok(ConsultOutcome::Failed { reason });
}
};
let reply_text = advisor_reply_text(&agg);
@@ -426,7 +445,9 @@ impl AdvisorGate {
reply_head: Some(reply_head),
usage: Some(&agg.usage),
});
- Ok(ConsultOutcome::Failed)
+ Ok(ConsultOutcome::Failed {
+ reason: "parse_error",
+ })
}
}
}
@@ -485,7 +506,7 @@ impl Algorithm for AdvisorGate {
enum ConsultOutcome {
Approve,
Redo { plan: String },
- Failed,
+ Failed { reason: &'static str },
}
fn algorithm_error(message: impl Into<String>) -> LibsyError {
diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rs
index 9d1916446..af75f7b64 100644
--- a/crates/libsy/src/algorithms/composite.rs
+++ b/crates/libsy/src/algorithms/composite.rs
@@ -90,6 +90,9 @@ impl Processor<State> for TierSetter {
if let Some(tier) = identity.and_then(|identity| self.tiers.lock().get(&identity).copied())
{
set_fall_open(state, tier);
+ if let Some(driver) = driver {
+ driver.set_evidence_if_empty(serde_json::json!({"source": "retained"}));
+ }
}
Ok(())
}
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs
index 593c22925..47523dc13 100644
--- a/crates/libsy/src/algorithms/escalation.rs
+++ b/crates/libsy/src/algorithms/escalation.rs
@@ -92,6 +92,10 @@ impl Classifier<State> for EscalationClassifier {
// A confirmed session stays capable without a judge call.
if streak(state) >= self.confirmations {
+ driver.set_evidence(serde_json::json!({
+ "source": "escalation",
+ "verdict": "latched",
+ }));
return Ok((decisive(&self.capable), None));
}
@@ -111,7 +115,13 @@ impl Classifier<State> for EscalationClassifier {
Err(LibsyError::ClientCall {
source: LlmClientError::ContextWindowExceeded { .. },
..
- }) => return Ok((decisive(&self.capable), None)),
+ }) => {
+ driver.set_evidence(serde_json::json!({
+ "source": "fallback",
+ "reason_code": "context_window",
+ }));
+ return Ok((decisive(&self.capable), None));
+ }
Err(e) => return Err(e),
};
// The call resolves when its stream handle arrives; transport can still fail while
@@ -119,6 +129,10 @@ impl Classifier<State> for EscalationClassifier {
let agg = match efficient_response.llm_response.into_agg().await {
Ok(agg) => agg,
Err(LlmClientError::Transport { .. }) => {
+ driver.set_evidence(serde_json::json!({
+ "source": "fallback",
+ "reason_code": "transport",
+ }));
return Ok((decisive(&self.capable), None));
}
Err(source) => {
@@ -158,9 +172,20 @@ impl Classifier<State> for EscalationClassifier {
if escalate && pending >= self.confirmations {
// Streak confirmed: drop the efficient response, caller will serve capable.
+ driver.set_evidence(serde_json::json!({
+ "source": "escalation",
+ "verdict": "escalate",
+ }));
return Ok((decisive(&self.capable), None));
}
+ if escalate {
+ driver.set_evidence(serde_json::json!({
+ "source": "escalation",
+ "verdict": "pending",
+ }));
+ }
+
Ok((decisive(&self.efficient), Some(efficient_response)))
}
}
diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs
index 2146d96fb..c2cebe68a 100644
--- a/crates/libsy/src/algorithms/fall_through.rs
+++ b/crates/libsy/src/algorithms/fall_through.rs
@@ -73,9 +73,12 @@ impl<S: Send> Classifier<S> for DefaultTarget {
&self,
_state: &mut S,
_request: &mut Request,
- _driver: Option<&Driver>,
+ driver: Option<&Driver>,
) -> Result<(Classification, Option<Response>)> {
// Zero confidence: this is a fallback, not a judgement.
+ if let Some(driver) = driver {
+ driver.set_evidence_if_empty(serde_json::json!({"source": "fall_open"}));
+ }
Ok((
Classification::Scores(vec![Score {
target: self.target.clone(),
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index 599e9f9e0..652d647cf 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -243,6 +243,29 @@ impl JudgePolicy for TaskClassifierPolicy {
}
}
+/// Maps valid verdicts to scores, invalid verdicts to a reason, and leaves absent verdicts alone.
+fn capability_evidence(
+ policy: &TaskClassifierPolicy,
+ verdict: Option<&TaskClassifierVerdict>,
+) -> Option<Value> {
+ let verdict = verdict?;
+ let Some(threshold) = verdict
+ .is_valid()
+ .then(|| policy.threshold(verdict))
+ .flatten()
+ else {
+ return Some(serde_json::json!({
+ "source": "fail_open",
+ "reason_code": "invalid_verdict",
+ }));
+ };
+ Some(serde_json::json!({
+ "source": "llm-classifier",
+ "score": verdict.p_solve,
+ "threshold": threshold,
+ }))
+}
+
#[derive(Clone, Debug)]
/// Settings that control capability classifier prompting and routing.
pub struct TaskClassifierConfig {
@@ -605,7 +628,8 @@ impl LlmTaskClassifier {
capable_target.clone(),
&config,
),
- ),
+ )
+ .with_evidence(capability_evidence),
capable_target: capable_target.clone(),
});
let inner: Arc<dyn Classifier<State>> = classifier.clone();
diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs
index 483a46282..3fd4acc1e 100644
--- a/crates/libsy/src/algorithms/stage.rs
+++ b/crates/libsy/src/algorithms/stage.rs
@@ -56,6 +56,11 @@ impl Classifier<State> for SourceStamp {
if let Some(winner) = classification.argmax(false)? {
record_decision_source(state, self.source);
record_routing_decision(self.source, &winner.target);
+ if let Some(driver) = driver {
+ driver.set_evidence_if_empty(serde_json::json!({
+ "source": self.source.as_str(),
+ }));
+ }
}
Ok((classification, served))
}
diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs
index 7c696aa4e..f557a28d6 100644
--- a/crates/libsy/src/algorithms/util/affinity.rs
+++ b/crates/libsy/src/algorithms/util/affinity.rs
@@ -232,7 +232,7 @@ where
&self,
_state: &mut S,
request: &mut Request,
- _driver: Option<&Driver>,
+ driver: Option<&Driver>,
) -> crate::Result<(Classification, Option<switchyard_protocol::Response>)> {
let Some(key) = self.affinity_key(request) else {
return Ok((Classification::Scores(Vec::new()), None));
@@ -243,6 +243,11 @@ where
return Ok((Classification::Scores(Vec::new()), None));
}
let assigned = self.assignments.lock().get(&key).cloned();
+ if assigned.is_some()
+ && let Some(driver) = driver
+ {
+ driver.set_evidence(serde_json::json!({"source": "retained"}));
+ }
Ok((
Classification::Scores(match assigned {
Some(target) => vec![Score {
diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs
index 54ded690c..00b47f98e 100644
--- a/crates/libsy/src/algorithms/util/escalation.rs
+++ b/crates/libsy/src/algorithms/util/escalation.rs
@@ -8,6 +8,7 @@
//! lives with the assembled algorithm in [`crate::algorithms::escalation`].
use serde::Deserialize;
+use serde_json::Value;
use switchyard_protocol::{ContentBlock, Message, ModelId, Role};
use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};
@@ -139,6 +140,19 @@ impl JudgePolicy for EscalationPolicy {
}
}
+/// Maps present verdicts to stable `escalate` or `continue` values; absent verdicts add nothing.
+fn escalation_evidence(
+ _policy: &EscalationPolicy,
+ verdict: Option<&EscalationVerdict>,
+) -> Option<Value> {
+ verdict.map(|verdict| {
+ serde_json::json!({
+ "source": "escalation",
+ "verdict": if verdict.escalate { "escalate" } else { "continue" },
+ })
+ })
+}
+
/// Builds the trajectory judge over `judge_target`, scoring `capable` when it escalates.
///
/// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value
@@ -163,7 +177,8 @@ pub(crate) fn build_judge(
),
judge_target,
EscalationPolicy { capable, efficient },
- ))
+ )
+ .with_evidence(escalation_evidence))
}
/// The 1-indexed model invocation the transcript ends on: one per assistant reply.
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index 0210bda27..a99fb82e5 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -202,11 +202,18 @@ pub trait JudgePolicy: Send + Sync {
fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification;
}
+type EvidenceFn<V, P> = fn(&P, Option<&V>) -> Option<Value>;
+
/// A classifier that calls one judge target and routes through its verdict policy.
-pub struct JudgeClassifier<J, P> {
+pub struct JudgeClassifier<J, P>
+where
+ J: Judge,
+ P: JudgePolicy<Verdict = J::Verdict>,
+{
judge: J,
target: ModelId,
policy: P,
+ evidence: Option<EvidenceFn<J::Verdict, P>>,
}
impl<J, P> JudgeClassifier<J, P>
@@ -220,6 +227,24 @@ where
judge,
target,
policy,
+ evidence: None,
+ }
+ }
+
+ /// Enables bounded evidence for built-in judges without widening the public policy trait.
+ pub(crate) fn with_evidence(mut self, evidence: EvidenceFn<J::Verdict, P>) -> Self {
+ self.evidence = Some(evidence);
+ self
+ }
+
+ /// Adds fail-open evidence only for evidence-enabled judges and preserves an earlier decision.
+ fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) {
+ report_fail_open(self.target.as_str(), error, reason);
+ if self.evidence.is_some() {
+ driver.set_evidence_if_empty(serde_json::json!({
+ "source": "fail_open",
+ "reason_code": reason,
+ }));
}
}
@@ -246,11 +271,7 @@ where
)
.await
.inspect_err(|error| {
- report_fail_open(
- judge_model,
- safe_error_summary(error),
- libsy_error_reason(error),
- )
+ self.report_fail_open(driver, safe_error_summary(error), libsy_error_reason(error));
})
.ok()?;
let aggregate = response
@@ -258,17 +279,13 @@ where
.into_agg()
.await
.inspect_err(|error| {
- report_fail_open(
- judge_model,
- safe_client_error(error),
- client_error_reason(error),
- )
+ self.report_fail_open(driver, safe_client_error(error), client_error_reason(error));
})
.ok()?;
self.judge
.parse(&aggregate)
.inspect_err(|error| {
- report_fail_open(judge_model, safe_error_summary(error), "parse_error")
+ self.report_fail_open(driver, safe_error_summary(error), "parse_error");
})
.ok()
}
@@ -333,8 +350,20 @@ where
});
};
let verdict = self.verdict(state, request, driver).await;
+ let classification = self.policy.to_classification(verdict.as_ref());
+ if let Some(evidence) = self
+ .evidence
+ .and_then(|evidence| evidence(&self.policy, verdict.as_ref()))
+ {
+ match &classification {
+ Classification::Scores(scores) if !scores.is_empty() => {
+ driver.set_evidence(evidence);
+ }
+ _ => driver.set_evidence_if_empty(evidence),
+ }
+ }
// A judge consultation is a side call, never the turn's answer.
- Ok((self.policy.to_classification(verdict.as_ref()), None))
+ Ok((classification, None))
}
}
diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs
index 44098293c..d5da79596 100644
--- a/crates/libsy/src/algorithms/util/stage.rs
+++ b/crates/libsy/src/algorithms/util/stage.rs
@@ -604,7 +604,7 @@ impl Classifier<State> for StageClassifier {
&self,
state: &mut State,
request: &mut Request,
- _driver: Option<&Driver>,
+ driver: Option<&Driver>,
) -> Result<(Classification, Option<switchyard_protocol::Response>)> {
let tool_signals = &state.tool_signals;
let Some(signal) = tool_signals else {
@@ -629,6 +629,17 @@ impl Classifier<State> for StageClassifier {
// is the only branch whose tier the signals actually chose — an
// ambiguous turn is decided further down the cascade.
self.apply_handoff_note(request, tier, source);
+ if let Some(driver) = driver {
+ let evidence = match (source, confidence) {
+ (DecisionSource::Dimensions, Some(confidence)) => serde_json::json!({
+ "source": source.as_str(),
+ "confidence": confidence,
+ "threshold": self.confidence_threshold,
+ }),
+ _ => serde_json::json!({"source": source.as_str()}),
+ };
+ driver.set_evidence(evidence);
+ }
// Prefer pick_tier's own confidence (e.g. 1.0 for an Override)
// over re-deriving it from the neutral 0.5 placeholder.
let conf = confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs());
diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs
index 0294dc158..10447b65e 100644
--- a/crates/libsy/src/algorithms/util/subagent.rs
+++ b/crates/libsy/src/algorithms/util/subagent.rs
@@ -123,7 +123,7 @@ where
&self,
_state: &mut S,
request: &mut Request,
- _driver: Option<&Driver>,
+ driver: Option<&Driver>,
) -> Result<(Classification, Option<Response>)> {
// Delegated *work* only. A harness maintenance turn (e.g. Codex `compact`) carries
// sub-agent lineage but is not delegated work, so it abstains and routes normally.
@@ -131,6 +131,9 @@ where
.metadata
.as_ref()
.is_some_and(Metadata::is_subagent_work);
+ if is_delegated_work && let Some(driver) = driver {
+ driver.set_evidence(serde_json::json!({"source": "subagent"}));
+ }
Ok((
Classification::Scores(if is_delegated_work {
vec![Score {
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 56e56f5fd..cecab86af 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -8,6 +8,8 @@ use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::In
use async_trait::async_trait;
use futures::{FutureExt, Stream, StreamExt};
+use parking_lot::Mutex;
+use serde_json::Value;
use tokio::sync::{mpsc, oneshot};
use tokio_stream::wrappers::ReceiverStream;
use tracing::Instrument;
@@ -121,6 +123,8 @@ pub struct Driver {
step_tx: mpsc::Sender<Result<Step>>,
/// The owning algorithm's telemetry label, stamped onto every call this driver publishes.
algorithm: String,
+ /// Run-scoped evidence shared by driver clones and attached only to a successful outcome.
+ evidence: Arc<Mutex<Option<Value>>>,
}
impl Driver {
@@ -136,11 +140,25 @@ impl Driver {
Self {
step_tx,
algorithm: algorithm.to_string(),
+ evidence: Arc::new(Mutex::new(None)),
},
step_rx,
)
}
+ /// Replace the current run's evidence when a component makes the final decision.
+ pub(crate) fn set_evidence(&self, evidence: Value) {
+ *self.evidence.lock() = Some(evidence);
+ }
+
+ /// Supply fallback evidence without replacing a decision made earlier in the cascade.
+ pub(crate) fn set_evidence_if_empty(&self, evidence: Value) {
+ let mut current = self.evidence.lock();
+ if current.is_none() {
+ *current = Some(evidence);
+ }
+ }
+
/// Publish a model call and await the consumer's response.
///
/// Errors if the stream is closed or the call failed.
@@ -204,9 +222,9 @@ impl Driver {
/// when the algorithm finishes.
pub(crate) async fn finish(&self, result: Result<RoutingOutcome>) -> Result<()> {
let result = result.map(|mut outcome| {
- let metadata = outcome
- .metadata
- .get_or_insert_with(|| crate::OutcomeMetadata::new(self.algorithm.clone(), None));
+ let metadata = outcome.metadata.get_or_insert_with(|| {
+ crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())
+ });
tracing::Span::current().record("outcome_id", metadata.outcome_id());
outcome
});
@@ -466,6 +484,8 @@ mod tests {
let response = driver
.call_model(request.clone(), vec![target.clone()])
.await?;
+ driver.set_evidence(serde_json::json!({"source": "test"}));
+ driver.set_evidence_if_empty(serde_json::json!({"source": "ignored"}));
Ok(RoutingOutcome::answered(target, request, response))
}
}
@@ -743,7 +763,10 @@ mod tests {
.get_version_num(),
7
);
- assert!(metadata.evidence.is_none());
+ assert_eq!(
+ metadata.evidence,
+ Some(serde_json::json!({"source": "test"}))
+ );
let response = outcome
.response
.ok_or_else(|| test_error("expected an answered outcome"))?;
diff --git a/crates/libsy/src/core/outcome_metadata.rs b/crates/libsy/src/core/outcome_metadata.rs
index 595e09961..ee8277727 100644
--- a/crates/libsy/src/core/outcome_metadata.rs
+++ b/crates/libsy/src/core/outcome_metadata.rs
@@ -3,19 +3,25 @@
//! Metadata describing a routing outcome.
+use serde_json::Value;
+
/// Identity and optional algorithm evidence attached to a successful routing outcome.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct OutcomeMetadata {
outcome_id: String,
/// Stable name of the algorithm that produced the outcome.
pub algorithm: String,
- /// Optional bounded, machine-readable evidence produced by the algorithm.
- pub evidence: Option<String>,
+ /// Optional algorithm-defined JSON evidence.
+ ///
+ /// Built-in algorithms emit an object with a stable `source` string and only the
+ /// relevant `score`, `confidence`, `threshold`, `verdict`, `trigger`, or `reason_code`
+ /// fields. Not every algorithm or decision produces evidence.
+ pub evidence: Option<Value>,
}
impl OutcomeMetadata {
/// Creates outcome metadata with a new UUIDv7 identifier.
- pub fn new(algorithm: String, evidence: Option<String>) -> Self {
+ pub fn new(algorithm: String, evidence: Option<Value>) -> Self {
Self {
outcome_id: uuid::Uuid::now_v7().to_string(),
algorithm,