Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): populate bounded outcome evidence (#655)
nachiketb-nvidiaf4152d076fd4
diff --git a/crates/libsy/src/algorithms/advisor_gate.rs b/crates/libsy/src/algorithms/advisor_gate.rsindex 87c15bd1f..72fc2a230 100644--- a/crates/libsy/src/algorithms/advisor_gate.rs+++ b/crates/libsy/src/algorithms/advisor_gate.rs@@ -270,14 +270,34 @@ impl AdvisorGate {.consult(driver, &request, review_tail.as_deref(), trigger_label).await{- Ok(ConsultOutcome::Approve) => Ok(RoutingOutcome::answered(- self.executor.clone(),- request,- turn.into_response(),- )),- Ok(ConsultOutcome::Redo { plan }) => Ok(self.redo(request, turn, &plan)),- Ok(ConsultOutcome::Failed) => {+ Ok(ConsultOutcome::Approve) => {+ driver.set_evidence(serde_json::json!({+ "source": "advisor",+ "verdict": "approve",+ "trigger": trigger_label,+ }));+ Ok(RoutingOutcome::answered(+ self.executor.clone(),+ request,+ turn.into_response(),+ ))+ }+ Ok(ConsultOutcome::Redo { plan }) => {+ driver.set_evidence(serde_json::json!({+ "source": "advisor",+ "verdict": "redo",+ "trigger": trigger_label,+ }));+ Ok(self.redo(request, turn, &plan))+ }+ Ok(ConsultOutcome::Failed { reason }) => {self.budget.refund_failure(scope);+ driver.set_evidence(serde_json::json!({+ "source": "advisor",+ "verdict": "fail_open",+ "trigger": trigger_label,+ "reason_code": reason,+ }));Ok(RoutingOutcome::answered(self.executor.clone(),request,@@ -361,9 +381,8 @@ impl AdvisorGate {let agg = match reply {Ok(agg) => agg,Err(error) => {- record_consult_failure(crate::algorithms::util::llm_judge::libsy_error_reason(- &error,- ));+ let reason = crate::algorithms::util::llm_judge::libsy_error_reason(&error);+ record_consult_failure(reason);if !self.config.fail_open {// Surface as an algorithm failure (5xx), never as the// advisor's own client error: a typed ContextWindowExceeded@@ -386,7 +405,7 @@ impl AdvisorGate {reply_head: None,usage: None,});- return Ok(ConsultOutcome::Failed);+ return Ok(ConsultOutcome::Failed { reason });}};let reply_text = advisor_reply_text(&agg);@@ -426,7 +445,9 @@ impl AdvisorGate {reply_head: Some(reply_head),usage: Some(&agg.usage),});- Ok(ConsultOutcome::Failed)+ Ok(ConsultOutcome::Failed {+ reason: "parse_error",+ })}}}@@ -485,7 +506,7 @@ impl Algorithm for AdvisorGate {enum ConsultOutcome {Approve,Redo { plan: String },- Failed,+ Failed { reason: &'static str },}fn algorithm_error(message: impl Into<String>) -> LibsyError {diff --git a/crates/libsy/src/algorithms/composite.rs b/crates/libsy/src/algorithms/composite.rsindex 9d1916446..af75f7b64 100644--- a/crates/libsy/src/algorithms/composite.rs+++ b/crates/libsy/src/algorithms/composite.rs@@ -90,6 +90,9 @@ impl Processor<State> for TierSetter {if let Some(tier) = identity.and_then(|identity| self.tiers.lock().get(&identity).copied()){set_fall_open(state, tier);+ if let Some(driver) = driver {+ driver.set_evidence_if_empty(serde_json::json!({"source": "retained"}));+ }}Ok(())}diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rsindex 593c22925..47523dc13 100644--- a/crates/libsy/src/algorithms/escalation.rs+++ b/crates/libsy/src/algorithms/escalation.rs@@ -92,6 +92,10 @@ impl Classifier<State> for EscalationClassifier {// A confirmed session stays capable without a judge call.if streak(state) >= self.confirmations {+ driver.set_evidence(serde_json::json!({+ "source": "escalation",+ "verdict": "latched",+ }));return Ok((decisive(&self.capable), None));}@@ -111,7 +115,13 @@ impl Classifier<State> for EscalationClassifier {Err(LibsyError::ClientCall {source: LlmClientError::ContextWindowExceeded { .. },..- }) => return Ok((decisive(&self.capable), None)),+ }) => {+ driver.set_evidence(serde_json::json!({+ "source": "fallback",+ "reason_code": "context_window",+ }));+ return Ok((decisive(&self.capable), None));+ }Err(e) => return Err(e),};// The call resolves when its stream handle arrives; transport can still fail while@@ -119,6 +129,10 @@ impl Classifier<State> for EscalationClassifier {let agg = match efficient_response.llm_response.into_agg().await {Ok(agg) => agg,Err(LlmClientError::Transport { .. }) => {+ driver.set_evidence(serde_json::json!({+ "source": "fallback",+ "reason_code": "transport",+ }));return Ok((decisive(&self.capable), None));}Err(source) => {@@ -158,9 +172,20 @@ impl Classifier<State> for EscalationClassifier {if escalate && pending >= self.confirmations {// Streak confirmed: drop the efficient response, caller will serve capable.+ driver.set_evidence(serde_json::json!({+ "source": "escalation",+ "verdict": "escalate",+ }));return Ok((decisive(&self.capable), None));}+ if escalate {+ driver.set_evidence(serde_json::json!({+ "source": "escalation",+ "verdict": "pending",+ }));+ }+Ok((decisive(&self.efficient), Some(efficient_response)))}}diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rsindex 2146d96fb..c2cebe68a 100644--- a/crates/libsy/src/algorithms/fall_through.rs+++ b/crates/libsy/src/algorithms/fall_through.rs@@ -73,9 +73,12 @@ impl<S: Send> Classifier<S> for DefaultTarget {&self,_state: &mut S,_request: &mut Request,- _driver: Option<&Driver>,+ driver: Option<&Driver>,) -> Result<(Classification, Option<Response>)> {// Zero confidence: this is a fallback, not a judgement.+ if let Some(driver) = driver {+ driver.set_evidence_if_empty(serde_json::json!({"source": "fall_open"}));+ }Ok((Classification::Scores(vec![Score {target: self.target.clone(),diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex 599e9f9e0..652d647cf 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -243,6 +243,29 @@ impl JudgePolicy for TaskClassifierPolicy {}}+/// Maps valid verdicts to scores, invalid verdicts to a reason, and leaves absent verdicts alone.+fn capability_evidence(+ policy: &TaskClassifierPolicy,+ verdict: Option<&TaskClassifierVerdict>,+) -> Option<Value> {+ let verdict = verdict?;+ let Some(threshold) = verdict+ .is_valid()+ .then(|| policy.threshold(verdict))+ .flatten()+ else {+ return Some(serde_json::json!({+ "source": "fail_open",+ "reason_code": "invalid_verdict",+ }));+ };+ Some(serde_json::json!({+ "source": "llm-classifier",+ "score": verdict.p_solve,+ "threshold": threshold,+ }))+}+#[derive(Clone, Debug)]/// Settings that control capability classifier prompting and routing.pub struct TaskClassifierConfig {@@ -605,7 +628,8 @@ impl LlmTaskClassifier {capable_target.clone(),&config,),- ),+ )+ .with_evidence(capability_evidence),capable_target: capable_target.clone(),});let inner: Arc<dyn Classifier<State>> = classifier.clone();diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rsindex 483a46282..3fd4acc1e 100644--- a/crates/libsy/src/algorithms/stage.rs+++ b/crates/libsy/src/algorithms/stage.rs@@ -56,6 +56,11 @@ impl Classifier<State> for SourceStamp {if let Some(winner) = classification.argmax(false)? {record_decision_source(state, self.source);record_routing_decision(self.source, &winner.target);+ if let Some(driver) = driver {+ driver.set_evidence_if_empty(serde_json::json!({+ "source": self.source.as_str(),+ }));+ }}Ok((classification, served))}diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rsindex 7c696aa4e..f557a28d6 100644--- a/crates/libsy/src/algorithms/util/affinity.rs+++ b/crates/libsy/src/algorithms/util/affinity.rs@@ -232,7 +232,7 @@ where&self,_state: &mut S,request: &mut Request,- _driver: Option<&Driver>,+ driver: Option<&Driver>,) -> crate::Result<(Classification, Option<switchyard_protocol::Response>)> {let Some(key) = self.affinity_key(request) else {return Ok((Classification::Scores(Vec::new()), None));@@ -243,6 +243,11 @@ wherereturn Ok((Classification::Scores(Vec::new()), None));}let assigned = self.assignments.lock().get(&key).cloned();+ if assigned.is_some()+ && let Some(driver) = driver+ {+ driver.set_evidence(serde_json::json!({"source": "retained"}));+ }Ok((Classification::Scores(match assigned {Some(target) => vec![Score {diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rsindex 54ded690c..00b47f98e 100644--- a/crates/libsy/src/algorithms/util/escalation.rs+++ b/crates/libsy/src/algorithms/util/escalation.rs@@ -8,6 +8,7 @@//! lives with the assembled algorithm in [`crate::algorithms::escalation`].use serde::Deserialize;+use serde_json::Value;use switchyard_protocol::{ContentBlock, Message, ModelId, Role};use super::classifier_contract::{ClassifierContract, ClassifierContractConfig};@@ -139,6 +140,19 @@ impl JudgePolicy for EscalationPolicy {}}+/// Maps present verdicts to stable `escalate` or `continue` values; absent verdicts add nothing.+fn escalation_evidence(+ _policy: &EscalationPolicy,+ verdict: Option<&EscalationVerdict>,+) -> Option<Value> {+ verdict.map(|verdict| {+ serde_json::json!({+ "source": "escalation",+ "verdict": if verdict.escalate { "escalate" } else { "continue" },+ })+ })+}+/// Builds the trajectory judge over `judge_target`, scoring `capable` when it escalates.////// Loads the packaged prompt and schema, so an unusable asset or an unusable `config` value@@ -163,7 +177,8 @@ pub(crate) fn build_judge(),judge_target,EscalationPolicy { capable, efficient },- ))+ )+ .with_evidence(escalation_evidence))}/// The 1-indexed model invocation the transcript ends on: one per assistant reply.diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex 0210bda27..a99fb82e5 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -202,11 +202,18 @@ pub trait JudgePolicy: Send + Sync {fn to_classification(&self, verdict: Option<&Self::Verdict>) -> Classification;}+type EvidenceFn<V, P> = fn(&P, Option<&V>) -> Option<Value>;+/// A classifier that calls one judge target and routes through its verdict policy.-pub struct JudgeClassifier<J, P> {+pub struct JudgeClassifier<J, P>+where+ J: Judge,+ P: JudgePolicy<Verdict = J::Verdict>,+{judge: J,target: ModelId,policy: P,+ evidence: Option<EvidenceFn<J::Verdict, P>>,}impl<J, P> JudgeClassifier<J, P>@@ -220,6 +227,24 @@ wherejudge,target,policy,+ evidence: None,+ }+ }++ /// Enables bounded evidence for built-in judges without widening the public policy trait.+ pub(crate) fn with_evidence(mut self, evidence: EvidenceFn<J::Verdict, P>) -> Self {+ self.evidence = Some(evidence);+ self+ }++ /// Adds fail-open evidence only for evidence-enabled judges and preserves an earlier decision.+ fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) {+ report_fail_open(self.target.as_str(), error, reason);+ if self.evidence.is_some() {+ driver.set_evidence_if_empty(serde_json::json!({+ "source": "fail_open",+ "reason_code": reason,+ }));}}@@ -246,11 +271,7 @@ where).await.inspect_err(|error| {- report_fail_open(- judge_model,- safe_error_summary(error),- libsy_error_reason(error),- )+ self.report_fail_open(driver, safe_error_summary(error), libsy_error_reason(error));}).ok()?;let aggregate = response@@ -258,17 +279,13 @@ where.into_agg().await.inspect_err(|error| {- report_fail_open(- judge_model,- safe_client_error(error),- client_error_reason(error),- )+ self.report_fail_open(driver, safe_client_error(error), client_error_reason(error));}).ok()?;self.judge.parse(&aggregate).inspect_err(|error| {- report_fail_open(judge_model, safe_error_summary(error), "parse_error")+ self.report_fail_open(driver, safe_error_summary(error), "parse_error");}).ok()}@@ -333,8 +350,20 @@ where});};let verdict = self.verdict(state, request, driver).await;+ let classification = self.policy.to_classification(verdict.as_ref());+ if let Some(evidence) = self+ .evidence+ .and_then(|evidence| evidence(&self.policy, verdict.as_ref()))+ {+ match &classification {+ Classification::Scores(scores) if !scores.is_empty() => {+ driver.set_evidence(evidence);+ }+ _ => driver.set_evidence_if_empty(evidence),+ }+ }// A judge consultation is a side call, never the turn's answer.- Ok((self.policy.to_classification(verdict.as_ref()), None))+ Ok((classification, None))}}diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rsindex 44098293c..d5da79596 100644--- a/crates/libsy/src/algorithms/util/stage.rs+++ b/crates/libsy/src/algorithms/util/stage.rs@@ -604,7 +604,7 @@ impl Classifier<State> for StageClassifier {&self,state: &mut State,request: &mut Request,- _driver: Option<&Driver>,+ driver: Option<&Driver>,) -> Result<(Classification, Option<switchyard_protocol::Response>)> {let tool_signals = &state.tool_signals;let Some(signal) = tool_signals else {@@ -629,6 +629,17 @@ impl Classifier<State> for StageClassifier {// is the only branch whose tier the signals actually chose — an// ambiguous turn is decided further down the cascade.self.apply_handoff_note(request, tier, source);+ if let Some(driver) = driver {+ let evidence = match (source, confidence) {+ (DecisionSource::Dimensions, Some(confidence)) => serde_json::json!({+ "source": source.as_str(),+ "confidence": confidence,+ "threshold": self.confidence_threshold,+ }),+ _ => serde_json::json!({"source": source.as_str()}),+ };+ driver.set_evidence(evidence);+ }// Prefer pick_tier's own confidence (e.g. 1.0 for an Override)// over re-deriving it from the neutral 0.5 placeholder.let conf = confidence.unwrap_or_else(|| 2.0 * (probability - 0.5).abs());diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rsindex 0294dc158..10447b65e 100644--- a/crates/libsy/src/algorithms/util/subagent.rs+++ b/crates/libsy/src/algorithms/util/subagent.rs@@ -123,7 +123,7 @@ where&self,_state: &mut S,request: &mut Request,- _driver: Option<&Driver>,+ driver: Option<&Driver>,) -> Result<(Classification, Option<Response>)> {// Delegated *work* only. A harness maintenance turn (e.g. Codex `compact`) carries// sub-agent lineage but is not delegated work, so it abstains and routes normally.@@ -131,6 +131,9 @@ where.metadata.as_ref().is_some_and(Metadata::is_subagent_work);+ if is_delegated_work && let Some(driver) = driver {+ driver.set_evidence(serde_json::json!({"source": "subagent"}));+ }Ok((Classification::Scores(if is_delegated_work {vec![Score {diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 56e56f5fd..cecab86af 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -8,6 +8,8 @@ use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::Inuse async_trait::async_trait;use futures::{FutureExt, Stream, StreamExt};+use parking_lot::Mutex;+use serde_json::Value;use tokio::sync::{mpsc, oneshot};use tokio_stream::wrappers::ReceiverStream;use tracing::Instrument;@@ -121,6 +123,8 @@ pub struct Driver {step_tx: mpsc::Sender<Result<Step>>,/// The owning algorithm's telemetry label, stamped onto every call this driver publishes.algorithm: String,+ /// Run-scoped evidence shared by driver clones and attached only to a successful outcome.+ evidence: Arc<Mutex<Option<Value>>>,}impl Driver {@@ -136,11 +140,25 @@ impl Driver {Self {step_tx,algorithm: algorithm.to_string(),+ evidence: Arc::new(Mutex::new(None)),},step_rx,)}+ /// Replace the current run's evidence when a component makes the final decision.+ pub(crate) fn set_evidence(&self, evidence: Value) {+ *self.evidence.lock() = Some(evidence);+ }++ /// Supply fallback evidence without replacing a decision made earlier in the cascade.+ pub(crate) fn set_evidence_if_empty(&self, evidence: Value) {+ let mut current = self.evidence.lock();+ if current.is_none() {+ *current = Some(evidence);+ }+ }+/// Publish a model call and await the consumer's response.////// Errors if the stream is closed or the call failed.@@ -204,9 +222,9 @@ impl Driver {/// when the algorithm finishes.pub(crate) async fn finish(&self, result: Result<RoutingOutcome>) -> Result<()> {let result = result.map(|mut outcome| {- let metadata = outcome- .metadata- .get_or_insert_with(|| crate::OutcomeMetadata::new(self.algorithm.clone(), None));+ let metadata = outcome.metadata.get_or_insert_with(|| {+ crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())+ });tracing::Span::current().record("outcome_id", metadata.outcome_id());outcome});@@ -466,6 +484,8 @@ mod tests {let response = driver.call_model(request.clone(), vec![target.clone()]).await?;+ driver.set_evidence(serde_json::json!({"source": "test"}));+ driver.set_evidence_if_empty(serde_json::json!({"source": "ignored"}));Ok(RoutingOutcome::answered(target, request, response))}}@@ -743,7 +763,10 @@ mod tests {.get_version_num(),7);- assert!(metadata.evidence.is_none());+ assert_eq!(+ metadata.evidence,+ Some(serde_json::json!({"source": "test"}))+ );let response = outcome.response.ok_or_else(|| test_error("expected an answered outcome"))?;diff --git a/crates/libsy/src/core/outcome_metadata.rs b/crates/libsy/src/core/outcome_metadata.rsindex 595e09961..ee8277727 100644--- a/crates/libsy/src/core/outcome_metadata.rs+++ b/crates/libsy/src/core/outcome_metadata.rs@@ -3,19 +3,25 @@//! Metadata describing a routing outcome.+use serde_json::Value;+/// Identity and optional algorithm evidence attached to a successful routing outcome.#[derive(Clone, Debug, PartialEq, Eq)]pub struct OutcomeMetadata {outcome_id: String,/// Stable name of the algorithm that produced the outcome.pub algorithm: String,- /// Optional bounded, machine-readable evidence produced by the algorithm.- pub evidence: Option<String>,+ /// Optional algorithm-defined JSON evidence.+ ///+ /// Built-in algorithms emit an object with a stable `source` string and only the+ /// relevant `score`, `confidence`, `threshold`, `verdict`, `trigger`, or `reason_code`+ /// fields. Not every algorithm or decision produces evidence.+ pub evidence: Option<Value>,}impl OutcomeMetadata {/// Creates outcome metadata with a new UUIDv7 identifier.- pub fn new(algorithm: String, evidence: Option<String>) -> Self {+ pub fn new(algorithm: String, evidence: Option<Value>) -> Self {Self {outcome_id: uuid::Uuid::now_v7().to_string(),algorithm,