Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(stats): retain judge selections in session stats (#1192)
ryan-lempka317dcdec5fea
diff --git a/crates/libsy/src/algorithms/escalation.rs b/crates/libsy/src/algorithms/escalation.rsindex 0eda17531..9279260dd 100644--- a/crates/libsy/src/algorithms/escalation.rs+++ b/crates/libsy/src/algorithms/escalation.rs@@ -365,6 +365,8 @@ impl Classifier for EscalationClassifier {.escalation_judge.verdict(state, &judge_request, driver, judge_models).await?;+ self.escalation_judge.classify_verdict(&verdict, driver)?;+ let verdict = verdict.ok();let held = count(state, STREAK_KEY);let held_category = category(state).map(str::to_string);@@ -645,8 +647,13 @@ mod tests {).await?;+ let metadata = outcome.metadata.expect("routing outcome metadata");+ assert_eq!(metadata.judgments.len(), 1);+ assert_eq!(metadata.judgments[0].target, Some("efficient".into()));+ assert_eq!(metadata.judgments[0].category.as_deref(), Some("efficient"));+ assert_eq!(metadata.judgments[0].evidence, metadata.evidence);assert_eq!(- outcome.metadata.and_then(|metadata| metadata.evidence),+ metadata.evidence,Some(serde_json::json!({"source": "escalation","verdict": "continue",diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rsindex d90bb4b99..66f2288cf 100644--- a/crates/libsy/src/algorithms/llm_class.rs+++ b/crates/libsy/src/algorithms/llm_class.rs@@ -1716,11 +1716,11 @@ mod tests {assert_eq!(outcome.selected_model_id()?, expected, "{name}");assert!(outcome.response.is_none());assert_eq!(outcome.request.llm_request.messages, original_messages);- let evidence = outcome- .metadata- .and_then(|metadata| metadata.evidence)- .expect("routing evidence");+ let metadata = outcome.metadata.expect("routing metadata");+ assert_eq!(metadata.judgments.len(), 1);+ let evidence = metadata.evidence.expect("routing evidence");if matches!(name, "above" | "equal" | "below") {+ assert_eq!(metadata.judgments[0].target.as_deref(), Some(expected));assert_eq!(evidence["source"], "decision_classifier");assert_eq!(evidence["verdict"], "relative_advantage");assert_eq!(evidence["threshold"], settings.cutoff);@@ -1738,6 +1738,13 @@ mod tests {}let retained = drive(router, request.clone(), models, &serve).await?;assert_eq!(retained.selected_model_id()?, expected);+ assert!(+ retained+ .metadata+ .expect("retained metadata")+ .judgments+ .is_empty()+ );assert_eq!(calls.load(Ordering::SeqCst),1,diff --git a/crates/libsy/src/algorithms/llm_class/decision.rs b/crates/libsy/src/algorithms/llm_class/decision.rsindex efc706a77..bf67f1fa1 100644--- a/crates/libsy/src/algorithms/llm_class/decision.rs+++ b/crates/libsy/src/algorithms/llm_class/decision.rs@@ -197,20 +197,22 @@ impl Classifier for DecisionClassifier {} else {(efficient, Category::Efficient)};- driver.set_evidence(json!({+ let evidence = json!({"source": "decision_classifier","verdict": if self.config.criteria.is_some() { "candidate_choice" } else { "relative_advantage" },+ "selected_candidate": self.config.candidate(target)?,+ "score_option": score_option,"score": score,"threshold": self.config.cutoff,- }));- Ok((- Classification::Scores(vec![Score {- target: target.clone(),- confidence: 1.0,- category: Some(category),- }]),- None,- ))+ });+ let selected = Score {+ target: target.clone(),+ confidence: 1.0,+ category: Some(category),+ };+ driver.record_judgment(Some(selected.clone()), Some(evidence.clone()));+ driver.set_evidence(evidence);+ Ok((Classification::Scores(vec![selected]), None))}}@@ -221,6 +223,8 @@ fn unavailable(reason: &'static str,) -> (Classification, Option<Response>) {report_fail_open(judge.as_str(), error, reason);- driver.set_evidence_if_empty(json!({"source": "fail_open", "reason_code": reason}));+ let evidence = json!({"source": "fail_open", "reason_code": reason});+ driver.record_judgment(None, Some(evidence.clone()));+ driver.set_evidence_if_empty(evidence);(Classification::Ambiguous(vec![]), None)}diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rsindex a244a49a7..a416623e8 100644--- a/crates/libsy/src/algorithms/util/llm_judge.rs+++ b/crates/libsy/src/algorithms/util/llm_judge.rs@@ -280,7 +280,12 @@ where}/// Adds fail-open evidence only for evidence-enabled judges and preserves an earlier decision.- fn report_fail_open(&self, driver: &Driver, error: String, reason: &'static str) {+ fn report_fail_open(+ &self,+ driver: &Driver,+ error: String,+ reason: &'static str,+ ) -> &'static str {let judge_target = driver.first_model_for(&Category::Judge).map(|c| c.as_str())@@ -292,25 +297,19 @@ where"reason_code": reason,}));}+ reason}- /// Consults the judge, yielding `None` when it is unavailable, unintelligible, or- /// over its route deadline.- ///- /// Errors delivered by the host and invalid verdicts are logged and folded into `None`- /// for the policy's fallback branch. The HTTP driver delivers client errors here when- /// error recovery is enabled; otherwise it stops the run. A route deadline covers the- /// whole consultation — the model call and the response drain — so a judge that- /// answers headers promptly and then stalls mid-stream cannot hold the turn past- /// `judge_deadline_ms` either. On expiry it follows the same recovery split: folded- /// into `None` when recovering, surfaced as a client timeout when not.+ /// Consults the judge, retaining a bounded reason when a consultation fails open.+ /// The deadline covers both the model call and response drain. Deadline errors+ /// stop the run when recovery is disabled.pub(crate) async fn verdict(&self,state: &mut State,request: &Request,driver: &Driver,judge_models: &[ModelId],- ) -> Result<Option<J::Verdict>> {+ ) -> Result<std::result::Result<J::Verdict, &'static str>> {let judge_model = judge_models.first().ok_or(LibsyError::NoTargets)?.as_str();tracing::info!(target = judge_model, "consulting llm judge");@@ -323,32 +322,23 @@ whereself.recover_errors,).await- .inspect_err(|error| {+ .map_err(|error| {self.report_fail_open(driver,- safe_error_summary(error),- libsy_error_reason(error),- );- })- .ok()?;- let aggregate = response- .llm_response- .into_agg()- .await- .inspect_err(|error| {- self.report_fail_open(- driver,- safe_client_error(error),- client_error_reason(error),- );- })- .ok()?;- self.judge- .parse(&aggregate)- .inspect_err(|error| {- self.report_fail_open(driver, safe_error_summary(error), "parse_error");- })- .ok()+ safe_error_summary(&error),+ libsy_error_reason(&error),+ )+ })?;+ let aggregate = response.llm_response.into_agg().await.map_err(|error| {+ self.report_fail_open(+ driver,+ safe_client_error(&error),+ client_error_reason(&error),+ )+ })?;+ self.judge.parse(&aggregate).map_err(|error| {+ self.report_fail_open(driver, safe_error_summary(&error), "parse_error")+ })};let verdict = match self.judge.deadline_ms() {None => consult.await,@@ -368,13 +358,38 @@ where},));}- None+ Err("deadline")}}}};Ok(verdict)}++ /// Classifies and records one judge consultation.+ pub(crate) fn classify_verdict(+ &self,+ verdict: &std::result::Result<J::Verdict, &'static str>,+ driver: &Driver,+ ) -> Result<Classification> {+ let classification = self+ .policy+ .to_classification(verdict.as_ref().ok(), driver)?;+ let evidence = self.evidence.and_then(|evidence| match verdict.as_ref() {+ Ok(verdict) => evidence(&self.policy, Some(verdict)),+ Err(reason) => Some(serde_json::json!({"source": "fail_open", "reason_code": reason})),+ });+ driver.record_judgment(classification.argmax(false)?, evidence.clone());+ if let Some(evidence) = evidence {+ match &classification {+ Classification::Scores(scores) if !scores.is_empty() => {+ driver.set_evidence(evidence);+ }+ _ => driver.set_evidence_if_empty(evidence),+ }+ }+ Ok(classification)+ }}/// Logs and counts a judge failure with a bounded label that excludes message content.@@ -433,18 +448,7 @@ where});}let verdict = self.verdict(state, request, driver, judge_models).await?;- let classification = self.policy.to_classification(verdict.as_ref(), driver)?;- if let Some(evidence) = self- .evidence- .and_then(|evidence| evidence(&self.policy, verdict.as_ref()))- {- match &classification {- Classification::Scores(scores) if !scores.is_empty() => {- driver.set_evidence(evidence);- }- _ => driver.set_evidence_if_empty(evidence),- }- }+ let classification = self.classify_verdict(&verdict, driver)?;// A judge consultation is a side call, never the turn's answer.Ok((classification, None))}@@ -653,7 +657,8 @@ mod tests {let models = RuntimeModels::new([(Category::Judge, vec![ModelId::from("judge")])].into());let (driver, step_rx) = Driver::new("test", Arc::new(models));let mut steps = tokio_stream::wrappers::ReceiverStream::new(step_rx);- let classifier = classifier();+ let classifier = classifier().with_evidence(|_, _| None);+ driver.set_evidence(serde_json::json!({"reason_code": "earlier"}));let mut state = State::default();let mut request = request();@@ -665,7 +670,26 @@ mod tests {let (classification, ()) =tokio::join!(classifier.score(&mut state, &mut request, &driver), serve);let (classification, _) = classification?;- selected(classification)+ let target = selected(classification)?;+ driver+ .finish(Ok(crate::RoutingOutcome::route_to(+ target.clone(),+ vec![],+ request,+ )))+ .await?;+ let Some(Ok(Step::Done(outcome))) = steps.next().await else {+ panic!("missing outcome");+ };+ let judgments = outcome.metadata.expect("metadata").judgments;+ assert_eq!(judgments.len(), 1);+ if target == "no-verdict" {+ let reason = judgments[0].evidence.as_ref().expect("failure reason")["reason_code"]+ .as_str()+ .expect("reason code");+ assert_ne!(reason, "earlier");+ }+ Ok(target)}#[tokio::test]diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 390bbef6e..0d0fa88da 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -348,6 +348,12 @@ impl AnswerSummary {}}+#[derive(Default)]+struct RunEvidence {+ outcome: Option<Value>,+ judgments: Vec<crate::JudgeDecision>,+}+/// An algorithm's handle for requesting external work and awaiting the host's typed replies.#[derive(Clone)]pub struct Driver {@@ -357,7 +363,7 @@ pub struct Driver {algorithm: String,/// Run-scoped evidence shared by driver clones and attached only to a successful outcome.- evidence: Arc<Mutex<Option<Value>>>,+ evidence: Arc<Mutex<RunEvidence>>,/// Caller-supplied routing state, read and updated when a request carries no session ID.state: Arc<Mutex<State>>,@@ -386,7 +392,7 @@ impl Driver {Self {step_tx,algorithm: algorithm.to_string(),- evidence: Arc::new(Mutex::new(None)),+ evidence: Arc::new(Mutex::new(RunEvidence::default())),state: Arc::new(Mutex::new(State::default())),models,scope: Scope::Parent,@@ -406,17 +412,28 @@ impl Driver {/// Replace the current run's evidence when a component makes the final decision.pub(crate) fn set_evidence(&self, evidence: Value) {- *self.evidence.lock() = Some(evidence);+ self.evidence.lock().outcome = Some(evidence);}/// Supply fallback evidence without replacing a decision made earlier in the cascade.pub(crate) fn set_evidence_if_empty(&self, evidence: Value) {let mut current = self.evidence.lock();- if current.is_none() {- *current = Some(evidence);+ if current.outcome.is_none() {+ current.outcome = Some(evidence);}}+ pub(crate) fn record_judgment(&self, selected: Option<crate::Score>, evidence: Option<Value>) {+ let (target, category) = selected+ .map(|score| (Some(score.target), score.category))+ .unwrap_or_default();+ self.evidence.lock().judgments.push(crate::JudgeDecision {+ target,+ category: category.map(|category| category.as_str().to_string()),+ evidence,+ });+ }+/// Publish a model call and await the consumer's response.////// Errors if the stream is closed or the call failed.@@ -569,9 +586,11 @@ impl Driver {pub(crate) async fn finish(&self, result: Result<RoutingOutcome>) -> Result<()> {let result = result.map(|mut outcome| {outcome.state = self.take_state();+ let evidence = std::mem::take(&mut *self.evidence.lock());let metadata = outcome.metadata.get_or_insert_with(|| {- crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())+ crate::OutcomeMetadata::new(self.algorithm.clone(), evidence.outcome)});+ metadata.judgments.extend(evidence.judgments);observability::record_outcome(metadata, &outcome.selected_model_ids);outcome});@@ -908,6 +927,14 @@ mod tests {let response = driver.call_model(request.clone(), vec![target.clone()]).await?;+ driver.record_judgment(+ Some(crate::Score {+ target: target.clone(),+ category: Some(Category::Capable),+ confidence: 1.0,+ }),+ Some(serde_json::json!({"source": "judge"})),+ );driver.set_evidence(serde_json::json!({"source": "test"}));driver.set_evidence_if_empty(serde_json::json!({"source": "ignored"}));Ok(RoutingOutcome::answered(target, request, response))@@ -1329,6 +1356,10 @@ mod tests {metadata.evidence,Some(serde_json::json!({"source": "test"})));+ assert_eq!(+ metadata.judgments[0].evidence,+ Some(serde_json::json!({"source": "judge"}))+ );let response = outcome.response.ok_or_else(|| test_error("expected an answered outcome"))?;diff --git a/crates/libsy/src/core/outcome_metadata.rs b/crates/libsy/src/core/outcome_metadata.rsindex 2ae10f106..580281992 100644--- a/crates/libsy/src/core/outcome_metadata.rs+++ b/crates/libsy/src/core/outcome_metadata.rs@@ -4,19 +4,33 @@//! Metadata describing a routing outcome.use serde_json::Value;+use switchyard_protocol::ModelId;++/// A judge's selection after applying its routing policy.+#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]+pub struct JudgeDecision {+ /// Selected target, or none when the judge abstains.+ pub target: Option<ModelId>,+ /// Category of the selected target, when known.+ pub category: Option<String>,+ /// Available scores, thresholds or failure reasons.+ pub evidence: Option<Value>,+}/// Identity and optional algorithm evidence attached to a successful routing outcome.-#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]+#[derive(Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)]pub struct OutcomeMetadata {outcome_id: String,/// Stable name of the algorithm that produced the outcome.pub algorithm: String,/// Optional algorithm-defined JSON evidence.///- /// Built-in algorithms emit an object with a stable `source` string and only the- /// relevant `score`, `confidence`, `threshold`, `verdict`, `trigger`, or `reason_code`- /// fields. Not every algorithm or decision produces evidence.+ /// Built-in algorithms include a `source` and relevant scores or decision details.+ /// Not every algorithm or decision produces evidence.pub evidence: Option<Value>,+ /// Judge selections retained separately from the final routing reason.+ #[serde(default, skip_serializing_if = "Vec::is_empty")]+ pub judgments: Vec<JudgeDecision>,}impl OutcomeMetadata {@@ -26,6 +40,7 @@ impl OutcomeMetadata {outcome_id: uuid::Uuid::now_v7().to_string(),algorithm,evidence,+ judgments: Vec::new(),}}diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rsindex 1ebc94567..8f71fffda 100644--- a/crates/libsy/src/lib.rs+++ b/crates/libsy/src/lib.rs@@ -10,7 +10,7 @@ pub use core::algorithm::{Step, StepStream, drive,};pub use core::classifier::{Classification, Classifier, Score};-pub use core::outcome_metadata::OutcomeMetadata;+pub use core::outcome_metadata::{JudgeDecision, OutcomeMetadata};pub use core::processor::{Event, Processor};pub use core::state::{State, StateValue};diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex d34b5222d..d1bb3c4cc 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -206,6 +206,19 @@ impl SharedRoutingLog {) -> std::io::Result<Option<routing_log::SessionStatsSnapshot>> {routing_log::snapshot(&self.path, session_id)}++ fn append_outcome(+ &self,+ context: &routing_log::RoutingLogContext,+ metadata: libsy::OutcomeMetadata,+ ) {+ if metadata.judgments.is_empty() {+ return;+ }+ if let Err(error) = self.writer.lock().append_outcome(context, metadata) {+ tracing::warn!(path = %self.path.display(), %error, "routing outcome append failed");+ }+ }}impl ServerState {@@ -453,7 +466,11 @@ fn stats_observer(classifier_log: Option<(SharedRoutingLog, routing_log::RoutingLogContext)>,) -> RunObserver {Arc::new(move |observation| match observation {- RunObservation::Outcome(_) => {}+ RunObservation::Outcome(metadata) => {+ if let Some((log, context)) = classifier_log.as_ref() {+ log.append_outcome(context, metadata);+ }+ }RunObservation::AnswerCall(call) => {let latency_ms = call.duration.as_secs_f64() * 1_000.0;if call.is_success {@@ -1861,6 +1878,25 @@ targets = ["only"]};observer(call("judge-model", false));observer(call("routed-model", true));+ observer(RunObservation::Outcome(libsy::OutcomeMetadata::new(+ "stage".into(),+ None,+ )));+ for (target, category) in [+ ("strong", switchyard_protocol::Category::Capable),+ ("weak", switchyard_protocol::Category::Efficient),+ ] {+ let mut outcome = libsy::OutcomeMetadata::new(+ "composite".into(),+ Some(serde_json::json!({"source": "stage"})),+ );+ outcome.judgments.push(libsy::JudgeDecision {+ target: Some(target.into()),+ category: Some(category.as_str().to_string()),+ evidence: None,+ });+ observer(RunObservation::Outcome(outcome));+ }let snapshot = log.snapshot_session("session-1")@@ -1871,7 +1907,19 @@ targets = ["only"]assert_eq!(snapshot["models"]["judge-model"]["prompt_tokens"], 100);assert_eq!(snapshot["models"]["judge-model"]["completion_tokens"], 7);assert!(snapshot["models"].get("routed-model").is_none());- let record: serde_json::Value = serde_json::from_str(&std::fs::read_to_string(&log.path)?)?;+ assert_eq!(snapshot["total_calls"], 1);+ assert_eq!(snapshot["outcomes"].as_array().expect("outcomes").len(), 2);+ assert_eq!(snapshot["outcomes"][0]["judgments"][0]["target"], "strong");+ assert_eq!(+ snapshot["outcomes"][1]["judgments"][0]["category"],+ "efficient"+ );+ assert_eq!(snapshot["outcomes"][1]["turn_id"], "turn-1");+ assert!(log.snapshot_session("another-session")?.is_none());+ let contents = std::fs::read_to_string(&log.path)?;+ assert_eq!(contents.lines().count(), 3);+ let record: serde_json::Value =+ serde_json::from_str(contents.lines().next().expect("call record"))?;assert_eq!(record["session_id"], "session-1");assert_eq!(record["turn_id"], "turn-1");Ok(())diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rsindex ecc5487ed..388005175 100644--- a/crates/switchyard-server/src/routing_log.rs+++ b/crates/switchyard-server/src/routing_log.rs@@ -11,6 +11,7 @@ use std::path::{Path, PathBuf};use std::time::SystemTime;use humantime::format_rfc3339_millis;+use libsy::OutcomeMetadata;use serde::{Deserialize, Serialize};use switchyard_protocol::{Metadata, ModelId, Usage};@@ -65,6 +66,7 @@ impl RoutingLog {.unwrap_or(0);let usage = token_usage(usage);let record = RoutingRecord {+ outcome: None,cost,ts: format_rfc3339_millis(SystemTime::now()).to_string().into(),route_id: context.route_id.into(),@@ -90,7 +92,29 @@ impl RoutingLog {reasoning_tokens: usage.reasoning_tokens,total_tokens: usage.prompt_tokens.saturating_add(usage.completion_tokens),};- let mut line = serde_json::to_vec(&record).map_err(std::io::Error::other)?;+ self.append_record(&record)+ }++ pub(crate) fn append_outcome(+ &mut self,+ context: &RoutingLogContext,+ metadata: OutcomeMetadata,+ ) -> std::io::Result<()> {+ self.append_record(&serde_json::json!({+ "session_id": context.session_id,+ "outcome": LoggedOutcome {+ ts: format_rfc3339_millis(SystemTime::now()).to_string(),+ route_id: context.route_id.clone(),+ turn_id: context.turn_id.clone(),+ agent_id: context.agent.agent_id.clone(),+ parent_agent_id: context.agent.parent_agent_id.clone(),+ metadata,+ },+ }))+ }++ fn append_record(&mut self, record: &impl Serialize) -> std::io::Result<()> {+ let mut line = serde_json::to_vec(record).map_err(std::io::Error::other)?;line.push(b'\n');self.file.write_all(&line)@@ -119,10 +143,10 @@ pub(crate) fn snapshot(let Ok(record) = serde_json::from_slice::<RoutingRecord>(&line) else {continue;};- snapshot.add_record(&record, session_id);+ snapshot.add_record(record, session_id);}snapshot.sum_totals();- Ok((snapshot.total_calls > 0).then_some(snapshot))+ Ok((snapshot.total_calls > 0 || !snapshot.outcomes.is_empty()).then_some(snapshot))}/// Which agent made a call. A child agent can share its parent's session id, so@@ -209,6 +233,8 @@ impl RoutingLogContext {#[derive(Default, Deserialize, Serialize)]#[serde(default)]struct RoutingRecord<'a> {+ #[serde(skip_serializing_if = "Option::is_none")]+ outcome: Option<LoggedOutcome>,#[serde(default)]cost: Option<RecordedCost>,ts: Cow<'a, str>,@@ -268,6 +294,21 @@ pub(crate) struct SessionStatsSnapshot {total_cache_creation_tokens: u64,total_completion_tokens: u64,models: BTreeMap<ModelId, SessionModelStats>,+ #[serde(skip_serializing_if = "Vec::is_empty")]+ outcomes: Vec<LoggedOutcome>,+}++#[derive(Deserialize, Serialize)]+struct LoggedOutcome {+ ts: String,+ route_id: String,+ turn_id: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ agent_id: Option<String>,+ #[serde(skip_serializing_if = "Option::is_none")]+ parent_agent_id: Option<String>,+ #[serde(flatten)]+ metadata: OutcomeMetadata,}#[derive(Default, Serialize)]@@ -281,10 +322,14 @@ struct SessionModelStats {}impl SessionStatsSnapshot {- fn add_record(&mut self, record: &RoutingRecord<'_>, session_id: &str) {+ fn add_record(&mut self, mut record: RoutingRecord<'_>, session_id: &str) {if record.session_id.as_deref() != Some(session_id) {return;}+ if let Some(outcome) = record.outcome.take() {+ self.outcomes.push(outcome);+ return;+ }let model = match record.model.as_ref() {"" => "unknown",model => model,diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rsindex 7261261d8..501b971e2 100644--- a/crates/switchyard-server/tests/server.rs+++ b/crates/switchyard-server/tests/server.rs@@ -2461,6 +2461,11 @@ escalation = {{ confirmations = 1 }}"metadata": {"algorithm": "llm_task_classifier","evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},+ "judgments": [{+ "target": "model/weak",+ "category": "efficient",+ "evidence": {"source": "llm-classifier", "score": 0.9, "threshold": 0.5},+ }],},}));diff --git a/docs/operations/local_evaluation.md b/docs/operations/local_evaluation.mdindex 6e14d5233..ad3bbc1a3 100644--- a/docs/operations/local_evaluation.md+++ b/docs/operations/local_evaluation.md@@ -146,6 +146,8 @@ The decision counters show one `fall_open` choice and one `override`. Here, `falFor per-request records, restart with `--routing-log-file routing.jsonl`. Each record includes the selected model and token usage. Decision reasons are reported by the stats endpoint. These checks confirm routing behavior, not task quality or savings.+With an unreleased build from main, `GET /v1/routing/session-stats?session_id=...` includes judge selections under `outcomes[].judgments`, with their targets, categories and available score evidence. Later Stage decisions leave these selections intact.+## Compare configurations- [Benchmark results](../benchmarks.md) link to model pools, results, and runnable profiles. For Terminal-Bench 2.1, the [escalation profile](../../benchmark/routing-profiles/tb21-escalation-opus-glm-deepseek.toml) includes its judge and reasoning settings. Its comments explain the endpoint differences from the measured runs.