Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(stats): retain judge selections in session stats (#1192)

ryan-lempka317dcdec5fea
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rs
index 0eda17531..9279260dd 100644
--- a/crates/libsy/src/algorithms/escalation.rs
+++ b/crates/libsy/src/algorithms/escalation.rs
@@ -365,6 +365,8 @@ impl Classifier for EscalationClassifier {
.escalation_judge
.verdict(state, &judge_request, driver, judge_models)
.await?;
+ self.escalation_judge.classify_verdict(&verdict, driver)?;
+ let verdict = verdict.ok();
let held = count(state, STREAK_KEY);
let held_category = category(state).map(str::to_string);
@@ -645,8 +647,13 @@ mod tests {
)
.await?;
+ let metadata = outcome.metadata.expect("routing outcome metadata");
+ assert_eq!(metadata.judgments.len(), 1);
+ assert_eq!(metadata.judgments[0].target, Some("efficient".into()));
+ assert_eq!(metadata.judgments[0].category.as_deref(), Some("efficient"));
+ assert_eq!(metadata.judgments[0].evidence, metadata.evidence);
assert_eq!(
- outcome.metadata.and_then(|metadata| metadata.evidence),
+ metadata.evidence,
Some(serde_json::json!({
"source": "escalation",
"verdict": "continue",
diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs
index d90bb4b99..66f2288cf 100644
--- a/crates/libsy/src/algorithms/llm_class.rs
+++ b/crates/libsy/src/algorithms/llm_class.rs
@@ -1716,11 +1716,11 @@ mod tests {
assert_eq!(outcome.selected_model_id()?, expected, "{name}");
assert!(outcome.response.is_none());
assert_eq!(outcome.request.llm_request.messages, original_messages);
- let evidence = outcome
- .metadata
- .and_then(|metadata| metadata.evidence)
- .expect("routing evidence");
+ let metadata = outcome.metadata.expect("routing metadata");
+ assert_eq!(metadata.judgments.len(), 1);
+ let evidence = metadata.evidence.expect("routing evidence");
if matches!(name, "above" | "equal" | "below") {
+ assert_eq!(metadata.judgments[0].target.as_deref(), Some(expected));
assert_eq!(evidence["source"], "decision_classifier");
assert_eq!(evidence["verdict"], "relative_advantage");
assert_eq!(evidence["threshold"], settings.cutoff);
@@ -1738,6 +1738,13 @@ mod tests {
}
let retained = drive(router, request.clone(), models, &serve).await?;
assert_eq!(retained.selected_model_id()?, expected);
+ assert!(
+ retained
+ .metadata
+ .expect("retained metadata")
+ .judgments
+ .is_empty()
+ );
assert_eq!(
calls.load(Ordering::SeqCst),
1,
diff --git a/crates/libsy/src/algorithms/llm_class/decision.rs b/crates/libsy/src/algorithms/llm_class/decision.rs
index efc706a77..bf67f1fa1 100644
--- a/crates/libsy/src/algorithms/llm_class/decision.rs
+++ b/crates/libsy/src/algorithms/llm_class/decision.rs
@@ -197,20 +197,22 @@ impl Classifier for DecisionClassifier {
} else {
(efficient, Category::Efficient)
};
- driver.set_evidence(json!({
+ let evidence = json!({
"source": "decision_classifier",
"verdict": if self.config.criteria.is_some() { "candidate_choice" } else { "relative_advantage" },
+ "selected_candidate": self.config.candidate(target)?,
+ "score_option": score_option,
"score": score,
"threshold": self.config.cutoff,
- }));
- Ok((
- Classification::Scores(vec![Score {
- target: target.clone(),
- confidence: 1.0,
- category: Some(category),
- }]),
- None,
- ))
+ });
+ let selected = Score {
+ target: target.clone(),
+ confidence: 1.0,
+ category: Some(category),
+ };
+ driver.record_judgment(Some(selected.clone()), Some(evidence.clone()));
+ driver.set_evidence(evidence);
+ Ok((Classification::Scores(vec![selected]), None))
}
}
@@ -221,6 +223,8 @@ fn unavailable(
reason: &'static str,
) -> (Classification, Option<Response>) {
report_fail_open(judge.as_str(), error, reason);
- driver.set_evidence_if_empty(json!({"source": "fail_open", "reason_code": reason}));
+ let evidence = json!({"source": "fail_open", "reason_code": reason});
+ driver.record_judgment(None, Some(evidence.clone()));
+ driver.set_evidence_if_empty(evidence);
(Classification::Ambiguous(vec![]), None)
}
diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs
index a244a49a7..a416623e8 100644
--- a/crates/libsy/src/algorithms/util/llm_judge.rs
+++ b/crates/libsy/src/algorithms/util/llm_judge.rs
@@ -280,7 +280,12 @@ where
}
/// Adds fail-open evidence only for evidence-enabled judges and preserves an earlier decision.
- fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) {
+ fn report_fail_open(
+ &self,
+ driver: &Driver,
+ error: String,
+ reason: &'static str,
+ ) -> &'static str {
let judge_target = driver
.first_model_for(&Category::Judge)
.map(|c| c.as_str())
@@ -292,25 +297,19 @@ where
"reason_code": reason,
}));
}
+ reason
}
- /// Consults the judge, yielding `None` when it is unavailable, unintelligible, or
- /// over its route deadline.
- ///
- /// Errors delivered by the host and invalid verdicts are logged and folded into `None`
- /// for the policy's fallback branch. The HTTP driver delivers client errors here when
- /// error recovery is enabled; otherwise it stops the run. A route deadline covers the
- /// whole consultation — the model call and the response drain — so a judge that
- /// answers headers promptly and then stalls mid-stream cannot hold the turn past
- /// `judge_deadline_ms` either. On expiry it follows the same recovery split: folded
- /// into `None` when recovering, surfaced as a client timeout when not.
+ /// Consults the judge, retaining a bounded reason when a consultation fails open.
+ /// The deadline covers both the model call and response drain. Deadline errors
+ /// stop the run when recovery is disabled.
pub(crate) async fn verdict(
&self,
state: &mut State,
request: &Request,
driver: &Driver,
judge_models: &[ModelId],
- ) -> Result<Option<J::Verdict>> {
+ ) -> Result<std::result::Result<J::Verdict, &'static str>> {
let judge_model = judge_models.first().ok_or(LibsyError::NoTargets)?.as_str();
tracing::info!(target = judge_model, "consulting llm judge");
@@ -323,32 +322,23 @@ where
self.recover_errors,
)
.await
- .inspect_err(|error| {
+ .map_err(|error| {
self.report_fail_open(
driver,
- safe_error_summary(error),
- libsy_error_reason(error),
- );
- })
- .ok()?;
- let aggregate = response
- .llm_response
- .into_agg()
- .await
- .inspect_err(|error| {
- self.report_fail_open(
- driver,
- safe_client_error(error),
- client_error_reason(error),
- );
- })
- .ok()?;
- self.judge
- .parse(&aggregate)
- .inspect_err(|error| {
- self.report_fail_open(driver, safe_error_summary(error), "parse_error");
- })
- .ok()
+ safe_error_summary(&error),
+ libsy_error_reason(&error),
+ )
+ })?;
+ let aggregate = response.llm_response.into_agg().await.map_err(|error| {
+ self.report_fail_open(
+ driver,
+ safe_client_error(&error),
+ client_error_reason(&error),
+ )
+ })?;
+ self.judge.parse(&aggregate).map_err(|error| {
+ self.report_fail_open(driver, safe_error_summary(&error), "parse_error")
+ })
};
let verdict = match self.judge.deadline_ms() {
None => consult.await,
@@ -368,13 +358,38 @@ where
},
));
}
- None
+ Err("deadline")
}
}
}
};
Ok(verdict)
}
+
+ /// Classifies and records one judge consultation.
+ pub(crate) fn classify_verdict(
+ &self,
+ verdict: &std::result::Result<J::Verdict, &'static str>,
+ driver: &Driver,
+ ) -> Result<Classification> {
+ let classification = self
+ .policy
+ .to_classification(verdict.as_ref().ok(), driver)?;
+ let evidence = self.evidence.and_then(|evidence| match verdict.as_ref() {
+ Ok(verdict) => evidence(&self.policy, Some(verdict)),
+ Err(reason) => Some(serde_json::json!({"source": "fail_open", "reason_code": reason})),
+ });
+ driver.record_judgment(classification.argmax(false)?, evidence.clone());
+ if let Some(evidence) = evidence {
+ match &classification {
+ Classification::Scores(scores) if !scores.is_empty() => {
+ driver.set_evidence(evidence);
+ }
+ _ => driver.set_evidence_if_empty(evidence),
+ }
+ }
+ Ok(classification)
+ }
}
/// Logs and counts a judge failure with a bounded label that excludes message content.
@@ -433,18 +448,7 @@ where
});
}
let verdict = self.verdict(state, request, driver, judge_models).await?;
- let classification = self.policy.to_classification(verdict.as_ref(), driver)?;
- if let Some(evidence) = self
- .evidence
- .and_then(|evidence| evidence(&self.policy, verdict.as_ref()))
- {
- match &classification {
- Classification::Scores(scores) if !scores.is_empty() => {
- driver.set_evidence(evidence);
- }
- _ => driver.set_evidence_if_empty(evidence),
- }
- }
+ let classification = self.classify_verdict(&verdict, driver)?;
// A judge consultation is a side call, never the turn's answer.
Ok((classification, None))
}
@@ -653,7 +657,8 @@ mod tests {
let models = RuntimeModels::new([(Category::Judge, vec![ModelId::from("judge")])].into());
let (driver, step_rx) = Driver::new("test", Arc::new(models));
let mut steps = tokio_stream::wrappers::ReceiverStream::new(step_rx);
- let classifier = classifier();
+ let classifier = classifier().with_evidence(|_, _| None);
+ driver.set_evidence(serde_json::json!({"reason_code": "earlier"}));
let mut state = State::default();
let mut request = request();
@@ -665,7 +670,26 @@ mod tests {
let (classification, ()) =
tokio::join!(classifier.score(&mut state, &mut request, &driver), serve);
let (classification, _) = classification?;
- selected(classification)
+ let target = selected(classification)?;
+ driver
+ .finish(Ok(crate::RoutingOutcome::route_to(
+ target.clone(),
+ vec![],
+ request,
+ )))
+ .await?;
+ let Some(Ok(Step::Done(outcome))) = steps.next().await else {
+ panic!("missing outcome");
+ };
+ let judgments = outcome.metadata.expect("metadata").judgments;
+ assert_eq!(judgments.len(), 1);
+ if target == "no-verdict" {
+ let reason = judgments[0].evidence.as_ref().expect("failure reason")["reason_code"]
+ .as_str()
+ .expect("reason code");
+ assert_ne!(reason, "earlier");
+ }
+ Ok(target)
}
#[tokio::test]
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 390bbef6e..0d0fa88da 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -348,6 +348,12 @@ impl AnswerSummary {
}
}
+#[derive(Default)]
+struct RunEvidence {
+ outcome: Option<Value>,
+ judgments: Vec<crate::JudgeDecision>,
+}
+
/// An algorithm's handle for requesting external work and awaiting the host's typed replies.
#[derive(Clone)]
pub struct Driver {
@@ -357,7 +363,7 @@ pub struct Driver {
algorithm: String,
/// Run-scoped evidence shared by driver clones and attached only to a successful outcome.
- evidence: Arc<Mutex<Option<Value>>>,
+ evidence: Arc<Mutex<RunEvidence>>,
/// Caller-supplied routing state, read and updated when a request carries no session ID.
state: Arc<Mutex<State>>,
@@ -386,7 +392,7 @@ impl Driver {
Self {
step_tx,
algorithm: algorithm.to_string(),
- evidence: Arc::new(Mutex::new(None)),
+ evidence: Arc::new(Mutex::new(RunEvidence::default())),
state: Arc::new(Mutex::new(State::default())),
models,
scope: Scope::Parent,
@@ -406,17 +412,28 @@ impl Driver {
/// Replace the current run's evidence when a component makes the final decision.
pub(crate) fn set_evidence(&self, evidence: Value) {
- *self.evidence.lock() = Some(evidence);
+ self.evidence.lock().outcome = Some(evidence);
}
/// Supply fallback evidence without replacing a decision made earlier in the cascade.
pub(crate) fn set_evidence_if_empty(&self, evidence: Value) {
let mut current = self.evidence.lock();
- if current.is_none() {
- *current = Some(evidence);
+ if current.outcome.is_none() {
+ current.outcome = Some(evidence);
}
}
+ pub(crate) fn record_judgment(&self, selected: Option<crate::Score>, evidence: Option<Value>) {
+ let (target, category) = selected
+ .map(|score| (Some(score.target), score.category))
+ .unwrap_or_default();
+ self.evidence.lock().judgments.push(crate::JudgeDecision {
+ target,
+ category: category.map(|category| category.as_str().to_string()),
+ evidence,
+ });
+ }
+
/// Publish a model call and await the consumer's response.
///
/// Errors if the stream is closed or the call failed.
@@ -569,9 +586,11 @@ impl Driver {
pub(crate) async fn finish(&self, result: Result<RoutingOutcome>) -> Result<()> {
let result = result.map(|mut outcome| {
outcome.state = self.take_state();
+ let evidence = std::mem::take(&mut *self.evidence.lock());
let metadata = outcome.metadata.get_or_insert_with(|| {
- crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())
+ crate::OutcomeMetadata::new(self.algorithm.clone(), evidence.outcome)
});
+ metadata.judgments.extend(evidence.judgments);
observability::record_outcome(metadata, &outcome.selected_model_ids);
outcome
});
@@ -908,6 +927,14 @@ mod tests {
let response = driver
.call_model(request.clone(), vec![target.clone()])
.await?;
+ driver.record_judgment(
+ Some(crate::Score {
+ target: target.clone(),
+ category: Some(Category::Capable),
+ confidence: 1.0,
+ }),
+ Some(serde_json::json!({"source": "judge"})),
+ );
driver.set_evidence(serde_json::json!({"source": "test"}));
driver.set_evidence_if_empty(serde_json::json!({"source": "ignored"}));
Ok(RoutingOutcome::answered(target, request, response))
@@ -1329,6 +1356,10 @@ mod tests {
metadata.evidence,
Some(serde_json::json!({"source": "test"}))
);
+ assert_eq!(
+ metadata.judgments[0].evidence,
+ Some(serde_json::json!({"source": "judge"}))
+ );
let response = outcome
.response
.ok_or_else(|| test_error("expected an answered outcome"))?;
diff --git a/crates/libsy/src/core/outcome_metadata.rs b/crates/libsy/src/core/outcome_metadata.rs
index 2ae10f106..580281992 100644
--- a/crates/libsy/src/core/outcome_metadata.rs
+++ b/crates/libsy/src/core/outcome_metadata.rs
@@ -4,19 +4,33 @@
//! Metadata describing a routing outcome.
use serde_json::Value;
+use switchyard_protocol::ModelId;
+
+/// A judge's selection after applying its routing policy.
+#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
+pub struct JudgeDecision {
+ /// Selected target, or none when the judge abstains.
+ pub target: Option<ModelId>,
+ /// Category of the selected target, when known.
+ pub category: Option<String>,
+ /// Available scores, thresholds or failure reasons.
+ pub evidence: Option<Value>,
+}
/// Identity and optional algorithm evidence attached to a successful routing outcome.
-#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
+#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
pub struct OutcomeMetadata {
outcome_id: String,
/// Stable name of the algorithm that produced the outcome.
pub algorithm: String,
/// Optional algorithm-defined JSON evidence.
///
- /// Built-in algorithms emit an object with a stable `source` string and only the
- /// relevant `score`, `confidence`, `threshold`, `verdict`, `trigger`, or `reason_code`
- /// fields. Not every algorithm or decision produces evidence.
+ /// Built-in algorithms include a `source` and relevant scores or decision details.
+ /// Not every algorithm or decision produces evidence.
pub evidence: Option<Value>,
+ /// Judge selections retained separately from the final routing reason.
+ #[serde(default, skip_serializing_if = "Vec::is_empty")]
+ pub judgments: Vec<JudgeDecision>,
}
impl OutcomeMetadata {
@@ -26,6 +40,7 @@ impl OutcomeMetadata {
outcome_id: uuid::Uuid::now_v7().to_string(),
algorithm,
evidence,
+ judgments: Vec::new(),
}
}
diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs
index 1ebc94567..8f71fffda 100644
--- a/crates/libsy/src/lib.rs
+++ b/crates/libsy/src/lib.rs
@@ -10,7 +10,7 @@ pub use core::algorithm::{
Step, StepStream, drive,
};
pub use core::classifier::{Classification, Classifier, Score};
-pub use core::outcome_metadata::OutcomeMetadata;
+pub use core::outcome_metadata::{JudgeDecision, OutcomeMetadata};
pub use core::processor::{Event, Processor};
pub use core::state::{State, StateValue};
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index d34b5222d..d1bb3c4cc 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -206,6 +206,19 @@ impl SharedRoutingLog {
) -> std::io::Result<Option<routing_log::SessionStatsSnapshot>> {
routing_log::snapshot(&self.path, session_id)
}
+
+ fn append_outcome(
+ &self,
+ context: &routing_log::RoutingLogContext,
+ metadata: libsy::OutcomeMetadata,
+ ) {
+ if metadata.judgments.is_empty() {
+ return;
+ }
+ if let Err(error) = self.writer.lock().append_outcome(context, metadata) {
+ tracing::warn!(path = %self.path.display(), %error, "routing outcome append failed");
+ }
+ }
}
impl ServerState {
@@ -453,7 +466,11 @@ fn stats_observer(
classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,
) -> RunObserver {
Arc::new(move |observation| match observation {
- RunObservation::Outcome(_) => {}
+ RunObservation::Outcome(metadata) => {
+ if let Some((log, context)) = classifier_log.as_ref() {
+ log.append_outcome(context, metadata);
+ }
+ }
RunObservation::AnswerCall(call) => {
let latency_ms = call.duration.as_secs_f64() * 1_000.0;
if call.is_success {
@@ -1861,6 +1878,25 @@ targets = ["only"]
};
observer(call("judge-model", false));
observer(call("routed-model", true));
+ observer(RunObservation::Outcome(libsy::OutcomeMetadata::new(
+ "stage".into(),
+ None,
+ )));
+ for (target, category) in [
+ ("strong", switchyard_protocol::Category::Capable),
+ ("weak", switchyard_protocol::Category::Efficient),
+ ] {
+ let mut outcome = libsy::OutcomeMetadata::new(
+ "composite".into(),
+ Some(serde_json::json!({"source": "stage"})),
+ );
+ outcome.judgments.push(libsy::JudgeDecision {
+ target: Some(target.into()),
+ category: Some(category.as_str().to_string()),
+ evidence: None,
+ });
+ observer(RunObservation::Outcome(outcome));
+ }
let snapshot = log
.snapshot_session("session-1")
@@ -1871,7 +1907,19 @@ targets = ["only"]
assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100);
assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7);
assert!(snapshot["models"].get("routed-model").is_none());
- let record: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&log.path)?)?;
+ assert_eq!(snapshot["total_calls"], 1);
+ assert_eq!(snapshot["outcomes"].as_array().expect("outcomes").len(), 2);
+ assert_eq!(snapshot["outcomes"][0]["judgments"][0]["target"], "strong");
+ assert_eq!(
+ snapshot["outcomes"][1]["judgments"][0]["category"],
+ "efficient"
+ );
+ assert_eq!(snapshot["outcomes"][1]["turn_id"], "turn-1");
+ assert!(log.snapshot_session("another-session")?.is_none());
+ let contents = std::fs::read_to_string(&log.path)?;
+ assert_eq!(contents.lines().count(), 3);
+ let record: serde_json::Value =
+ serde_json::from_str(contents.lines().next().expect("call record"))?;
assert_eq!(record["session_id"], "session-1");
assert_eq!(record["turn_id"], "turn-1");
Ok(())
diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs
index ecc5487ed..388005175 100644
--- a/crates/switchyard-server/src/routing_log.rs
+++ b/crates/switchyard-server/src/routing_log.rs
@@ -11,6 +11,7 @@ use std::path::{Path, PathBuf};
use std::time::SystemTime;
use humantime::format_rfc3339_millis;
+use libsy::OutcomeMetadata;
use serde::{Deserialize, Serialize};
use switchyard_protocol::{Metadata, ModelId, Usage};
@@ -65,6 +66,7 @@ impl RoutingLog {
.unwrap_or(0);
let usage = token_usage(usage);
let record = RoutingRecord {
+ outcome: None,
cost,
ts: format_rfc3339_millis(SystemTime::now()).to_string().into(),
route_id: context.route_id.into(),
@@ -90,7 +92,29 @@ impl RoutingLog {
reasoning_tokens: usage.reasoning_tokens,
total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens),
};
- let mut line = serde_json::to_vec(&record).map_err(std::io::Error::other)?;
+ self.append_record(&record)
+ }
+
+ pub(crate) fn append_outcome(
+ &mut self,
+ context: &RoutingLogContext,
+ metadata: OutcomeMetadata,
+ ) -> std::io::Result<()> {
+ self.append_record(&serde_json::json!({
+ "session_id": context.session_id,
+ "outcome": LoggedOutcome {
+ ts: format_rfc3339_millis(SystemTime::now()).to_string(),
+ route_id: context.route_id.clone(),
+ turn_id: context.turn_id.clone(),
+ agent_id: context.agent.agent_id.clone(),
+ parent_agent_id: context.agent.parent_agent_id.clone(),
+ metadata,
+ },
+ }))
+ }
+
+ fn append_record(&mut self, record: &impl Serialize) -> std::io::Result<()> {
+ let mut line = serde_json::to_vec(record).map_err(std::io::Error::other)?;
line.push(b'\n');
self.file.write_all(&line)
@@ -119,10 +143,10 @@ pub(crate) fn snapshot(
let Ok(record) = serde_json::from_slice::<RoutingRecord>(&line) else {
continue;
};
- snapshot.add_record(&record, session_id);
+ snapshot.add_record(record, session_id);
}
snapshot.sum_totals();
- Ok((snapshot.total_calls > 0).then_some(snapshot))
+ Ok((snapshot.total_calls > 0 || !snapshot.outcomes.is_empty()).then_some(snapshot))
}
/// Which agent made a call. A child agent can share its parent's session id, so
@@ -209,6 +233,8 @@ impl RoutingLogContext {
#[derive(Default, Deserialize, Serialize)]
#[serde(default)]
struct RoutingRecord<'a> {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ outcome: Option<LoggedOutcome>,
#[serde(default)]
cost: Option<RecordedCost>,
ts: Cow<'a, str>,
@@ -268,6 +294,21 @@ pub(crate) struct SessionStatsSnapshot {
total_cache_creation_tokens: u64,
total_completion_tokens: u64,
models: BTreeMap<ModelId, SessionModelStats>,
+ #[serde(skip_serializing_if = "Vec::is_empty")]
+ outcomes: Vec<LoggedOutcome>,
+}
+
+#[derive(Deserialize, Serialize)]
+struct LoggedOutcome {
+ ts: String,
+ route_id: String,
+ turn_id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ agent_id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ parent_agent_id: Option<String>,
+ #[serde(flatten)]
+ metadata: OutcomeMetadata,
}
#[derive(Default, Serialize)]
@@ -281,10 +322,14 @@ struct SessionModelStats {
}
impl SessionStatsSnapshot {
- fn add_record(&mut self, record: &RoutingRecord<'_>, session_id: &str) {
+ fn add_record(&mut self, mut record: RoutingRecord<'_>, session_id: &str) {
if record.session_id.as_deref() != Some(session_id) {
return;
}
+ if let Some(outcome) = record.outcome.take() {
+ self.outcomes.push(outcome);
+ return;
+ }
let model = match record.model.as_ref() {
"" => "unknown",
model => model,
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index 7261261d8..501b971e2 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -2461,6 +2461,11 @@ escalation = {{ confirmations = 1 }}
"metadata": {
"algorithm": "llm_task_classifier",
"evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},
+ "judgments": [{
+ "target": "model/weak",
+ "category": "efficient",
+ "evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},
+ }],
},
})
);
diff --git a/docs/operations/local_evaluation.md b/docs/operations/local_evaluation.md
index 6e14d5233..ad3bbc1a3 100644
--- a/docs/operations/local_evaluation.md
+++ b/docs/operations/local_evaluation.md
@@ -146,6 +146,8 @@ The decision counters show one `fall_open` choice and one `override`. Here, `fal
For per-request records, restart with `--routing-log-file routing.jsonl`. Each record includes the selected model and token usage. Decision reasons are reported by the stats endpoint. These checks confirm routing behavior, not task quality or savings.
+With an unreleased build from main, `GET /v1/routing/session-stats?session_id=...` includes judge selections under `outcomes[].judgments`, with their targets, categories and available score evidence. Later Stage decisions leave these selections intact.
+
## Compare configurations
- [Benchmark results](../benchmarks.md) link to model pools, results, and runnable profiles. For Terminal-Bench 2.1, the [escalation profile](../../benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml) includes its judge and reasoning settings. Its comments explain the endpoint differences from the measured runs.