Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(libsy): expose outcome metadata in OpenTelemetry and Python (#658)
nachiketb-nvidia578e1b7cbd9d
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rsindex 4d3921afa..8ae0a551e 100644--- a/crates/libsy-llm-client/tests/observability.rs+++ b/crates/libsy-llm-client/tests/observability.rs@@ -488,6 +488,7 @@ impl RoutedLlmClient for UsageClient {struct SingleCallAlgo {name: String,target_set: Vec<String>,+ metadata: Option<switchyard_libsy::OutcomeMetadata>,}#[async_trait]@@ -507,7 +508,19 @@ impl Algorithm for SingleCallAlgo {.ok_or(LibsyError::NoTargets)?.clone();tracing::info!("picked '{target}'");- Ok(RoutingOutcome::route_to(target.into(), Vec::new(), request))+ // Include configured fallbacks so the exported model order can be checked.+ let mut outcome = RoutingOutcome::route_to(+ target.into(),+ self.target_set+ .iter()+ .skip(1)+ .cloned()+ .map(ModelId::from)+ .collect(),+ request,+ );+ outcome.metadata = self.metadata.clone();+ Ok(outcome)}}@@ -561,6 +574,7 @@ fn algo(name: &str, model: &str) -> Arc<dyn Algorithm> {Arc::new(SingleCallAlgo {name: name.to_string(),target_set: vec![model.to_string()],+ metadata: None,})}@@ -748,7 +762,8 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback()}#[tokio::test]-async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_libsy::Result<()> {+async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchyard_libsy::Result<()>+{let _guard = serialize_test().lock().await;let (store, exporter, provider, span_exporter, _) = telemetry();const ALGO: &str = "obs-success-algo";@@ -775,7 +790,20 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lrequest.llm_request.output.max_output_tokens = Some(512);request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));request.llm_request.reasoning.effort = Some("high".to_string());- let (selected_model, _response) = run(algo(ALGO, MODEL), client, request).await?;+ let metadata = switchyard_libsy::OutcomeMetadata::new(+ ALGO.to_string(),+ Some(json!({+ "source": "llm-classifier", "score": 0.9, "threshold": 0.5,+ "verdict": "continue", "trigger": "turn", "reason_code": "test",+ "confidence": "wrong type", "unknown": LEAKED_CONTENT,+ })),+ );+ let algorithm = Arc::new(SingleCallAlgo {+ name: ALGO.to_string(),+ target_set: vec![MODEL.to_string(), "obs-fallback-model".to_string()],+ metadata: Some(metadata.clone()),+ });+ let (selected_model, _response) = run(algorithm, client, request).await?;assert_eq!(selected_model, MODEL);// Metrics: run/call counters and latency histograms keyed by algorithm,@@ -843,6 +871,51 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l// llm_call span carrying the selection, outcome, and token counts.let spans = store.spans();let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO);+ assert_eq!(+ spans+ .iter()+ .filter(|span| span.name == "libsy.run"+ && span.fields.get("algorithm").map(String::as_str) == Some(ALGO))+ .count(),+ 1+ );+ for (field, expected) in [+ ("outcome_id", metadata.outcome_id()),+ ("evidence.source", "llm-classifier"),+ ("evidence.score", "0.9"),+ ("evidence.threshold", "0.5"),+ ("evidence.verdict", "continue"),+ ("evidence.trigger", "turn"),+ ("evidence.reason_code", "test"),+ ] {+ assert_eq!(+ run_span.fields.get(field).map(String::as_str),+ Some(expected),+ "{field}"+ );+ }+ assert!(!run_span.fields.contains_key("evidence.confidence"));+ assert!(!format!("{run_span:?}").contains(LEAKED_CONTENT));+ let exported = span_exporter.get_finished_spans().expect("exported spans");+ let exported_run = exported+ .iter()+ .find(|span| {+ span.name == "libsy.run"+ && otel_attribute(span, "outcome_id")+ .is_some_and(|id| id.as_str() == metadata.outcome_id())+ })+ .expect("outcome span exported");+ assert_eq!(+ otel_attribute(exported_run, "evidence.score"),+ Some(&OtelValue::F64(0.9))+ );+ assert_eq!(+ otel_attribute(exported_run, "selected_model_ids"),+ Some(&OtelValue::Array(OtelArray::String(vec![+ MODEL.into(),+ "obs-fallback-model".into()+ ]),))+ );assert_eq!(run_span.parent, None);assert_eq!(run_span.fields.get("session_id").map(String::as_str),@@ -880,13 +953,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lrun_span.fields.get("agent_role").map(String::as_str),Some("reviewer"));- // Host-defined labels ride in generically via Metadata.extra_metadata.- assert!(- run_span- .fields- .get("extra_metadata")- .is_some_and(|extra| extra.contains("tenant") && extra.contains("obs-tenant-1"))- );+ assert!(!run_span.fields.contains_key("extra_metadata"));// The default-client serve inside `run` gets its own client-call span.let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);@@ -950,6 +1017,12 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l// The algorithm logs why it made the decision.let events = store.events();+ assert!(+ !events.iter().any(+ |event| event.fields.get("algorithm").map(String::as_str) == Some(ALGO)+ && event.fields.get("message").map(String::as_str) == Some("routing decision")+ )+ );assert!(events.iter().any(|event| {event.level == "INFO"@@ -1250,7 +1323,7 @@ async fn upstream_body_is_redacted_from_the_client_call_span() -> switchyard_lib}#[tokio::test]-async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Result<()> {+async fn failed_call_records_metrics_without_error_details() -> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;let (store, exporter, provider, _, _) = telemetry();const ALGO: &str = "obs-failure-algo";@@ -1272,7 +1345,10 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Ok(Step::Done(_)) => {return Err(test_error("expected the failed call to fail the run"));}- Err(_) => saw_error_step = true,+ Err(error) => {+ assert!(error.to_string().contains("synthetic upstream failure"));+ saw_error_step = true;+ }}}assert!(@@ -1307,51 +1383,27 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::None);- // Spans: both spans carry outcome=error and the propagated error text.+ // Error details remain with the caller, while operational status is preserved.let spans = store.spans();let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO);assert_eq!(run_span.fields.get("outcome").map(String::as_str),Some("error"));- assert!(- run_span- .fields- .get("error")- .is_some_and(|error| error.contains("synthetic upstream failure"))- );+ assert!(!run_span.fields.contains_key("error"));+ assert!(!run_span.fields.contains_key("outcome_id"));let call_span = find_span(&spans, "libsy.llm_call", "selected_model", MODEL);assert_eq!(call_span.fields.get("outcome").map(String::as_str),Some("error"));- // Structured logs warn once for the failed call and failed run.+ assert_eq!(call_span.parent.as_deref(), Some("libsy.run"));+ assert!(!call_span.fields.contains_key("error"));+let events = store.events();- assert!(- events.iter().any(|event| {- event.target == "libsy"- && event.level == "WARN"- && event.fields.get("selected_model").map(String::as_str) == Some(MODEL)- && event- .fields- .get("message")- .is_some_and(|message| message.contains("model call failed"))- }),- "no call-failure log for {MODEL} in {events:?}"- );- assert!(- events.iter().any(|event| {- event.target == "libsy"- && event.level == "WARN"- && event.fields.get("algorithm").map(String::as_str) == Some(ALGO)- && event- .fields- .get("message")- .is_some_and(|message| message.contains("algorithm run failed"))- }),- "no run-failure log for {ALGO} in {events:?}"- );+ assert!(!events.iter().any(|event| event.target == "libsy"+ && event.fields.get("algorithm").map(String::as_str) == Some(ALGO)));Ok(())}diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex cecab86af..6e01c20c7 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -176,7 +176,6 @@ impl Driver {selected_model = %models.first().map(ModelId::as_str).unwrap_or("NoTargets"),openinference.span.kind = "CHAIN",outcome = tracing::field::Empty,- error = tracing::field::Empty,input_tokens = tracing::field::Empty,output_tokens = tracing::field::Empty,total_tokens = tracing::field::Empty,@@ -225,7 +224,7 @@ impl Driver {let metadata = outcome.metadata.get_or_insert_with(|| {crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())});- tracing::Span::current().record("outcome_id", metadata.outcome_id());+ observability::record_outcome(metadata, &outcome.selected_model_ids);outcome});let selected_model = result@@ -383,9 +382,20 @@ impl RoutingIdentity {/// # Observability////// [`run_stream`](Self::run_stream) creates a `libsy.run` span, and each offloaded model-/// call creates a `libsy.llm_call` span. Routing decisions and failures are emitted through-/// `tracing`; metrics use the global OpenTelemetry meter provider. The provider call-/// itself belongs to the host, and is instrumented by whoever makes it.+/// call creates a nested `libsy.llm_call` span. Successful outcomes record their+/// [`OutcomeMetadata::outcome_id`](crate::OutcomeMetadata::outcome_id) on `libsy.run`,+/// alongside `selected_model_ids` (an ordered OpenTelemetry string array).+/// `algorithm` and `switchyard.algorithm` retain the run's [`Algorithm::name`].+/// Optional `evidence.source`, `evidence.verdict`, `evidence.trigger`, and+/// `evidence.reason_code` are strings; `evidence.score`, `evidence.confidence`, and+/// `evidence.threshold` are numbers. Unknown evidence fields are not exported.+/// These fields are span attributes, never metric labels.+///+/// The run/call observability helpers retain `outcome` status and operational metrics,+/// but omit error details and arbitrary request extra metadata. Algorithms and hosts+/// may emit their own logs. Errors still reach the caller unchanged.+/// The host controls the tracing subscriber and global OpenTelemetry+/// meter provider; libsy installs no exporter and performs no telemetry network I/O.#[async_trait]pub trait Algorithm: Send + Sync + 'static {/// Stable, low-cardinality name identifying this algorithm — thediff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex 763d31153..69c6d9295 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -38,8 +38,9 @@ use std::time::{Duration, Instant};use opentelemetry::metrics::Meter;use opentelemetry::{KeyValue, global};use tracing::Span;+use tracing_opentelemetry::OpenTelemetrySpanExt;-use crate::Result;+use crate::{OutcomeMetadata, Result};use switchyard_protocol::{ModelId, Request, Response};const METRICS_SCOPE: &str = "switchyard";@@ -61,16 +62,22 @@ pub(crate) fn outcome_value<T>(result: &Result<T>) -> &'static str {/// Span covering one algorithm run (the whole `route` execution).////// Correlation ids from the request [`switchyard_protocol::Metadata`] are recorded as span fields-/// when present. `tracing` spans cannot grow field names at runtime, so-/// arbitrary host labels ride in via [`switchyard_protocol::Metadata::extra_metadata`], recorded-/// whole into the `extra_metadata` field. `outcome` and `error` are filled in-/// by [`record_run`] when the run ends.+/// when present. Arbitrary extra metadata and error details are not exported.+/// [`record_outcome`] fills in successful outcome fields; [`record_run`] records+/// whether the run succeeded.pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {let span = tracing::info_span!(target: TRACING_TARGET,"libsy.run",algorithm,outcome_id = tracing::field::Empty,+ evidence.source = tracing::field::Empty,+ evidence.score = tracing::field::Empty,+ evidence.confidence = tracing::field::Empty,+ evidence.threshold = tracing::field::Empty,+ evidence.verdict = tracing::field::Empty,+ evidence.trigger = tracing::field::Empty,+ evidence.reason_code = tracing::field::Empty,switchyard.algorithm = algorithm,openinference.span.kind = "CHAIN",switchyard.route = tracing::field::Empty,@@ -81,9 +88,7 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {task_kind = tracing::field::Empty,agent_role = tracing::field::Empty,correlation_id = tracing::field::Empty,- extra_metadata = tracing::field::Empty,outcome = tracing::field::Empty,- error = tracing::field::Empty,);if let Some(route) = request.model_id() {span.record("switchyard.route", route.as_ref());@@ -104,13 +109,48 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {if let Some(session_id) = &metadata.session_id {span.record("session.id", session_id.as_str());}- if let Some(extra) = &metadata.extra_metadata {- span.record("extra_metadata", tracing::field::debug(extra));- }}span}+/// Projects a successful outcome onto the existing run span. Model IDs are an+/// ordered OpenTelemetry string array, preserving fallback order. Evidence uses typed fields;+/// unknown keys and values of the wrong type are omitted.+pub(crate) fn record_outcome(metadata: &OutcomeMetadata, models: &[ModelId]) {+ let span = Span::current();+ span.record("outcome_id", metadata.outcome_id());+ span.set_attribute(+ "selected_model_ids",+ opentelemetry::Value::Array(opentelemetry::Array::String(+ models+ .iter()+ .map(|model| model.to_string().into())+ .collect(),+ )),+ );+ if let Some(evidence) = &metadata.evidence {+ for (key, field) in [+ ("source", "evidence.source"),+ ("verdict", "evidence.verdict"),+ ("trigger", "evidence.trigger"),+ ("reason_code", "evidence.reason_code"),+ ] {+ if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_str) {+ span.record(field, value);+ }+ }+ for (key, field) in [+ ("score", "evidence.score"),+ ("confidence", "evidence.confidence"),+ ("threshold", "evidence.threshold"),+ ] {+ if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_f64) {+ span.record(field, value);+ }+ }+ }+}+/// Holds `switchyard.algorithms_in_flight` up by one for as long as it lives.struct InFlightRun {algorithm: String,@@ -141,7 +181,7 @@ fn record_algorithms_in_flight(algorithm: &str, delta: i64) {}/// Runs one algorithm task to completion, recording the run counter, duration-/// histogram, span outcome, and failure log when it resolves. Counts the run as+/// histogram and span outcome when it resolves. Counts the run as/// in flight for its whole duration./// Executes inside the `libsy.run` span its caller instruments the task with.pub(crate) async fn observe_run<T>(@@ -158,20 +198,10 @@ pub(crate) async fn observe_run<T>(}/// Records the end of one algorithm run: the run counter and duration-/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the-/// run failed.+/// histogram and the `outcome` field on `span`, without error details.fn record_run<T>(algorithm: &str, duration: Duration, result: &Result<T>, span: &Span) {let outcome = outcome_value(result);span.record("outcome", outcome);- if let Err(error) = result {- span.record("error", tracing::field::display(error));- tracing::warn!(- target: TRACING_TARGET,- algorithm,- error = %error,- "algorithm run failed"- );- }let attributes = [KeyValue::new("algorithm", algorithm.to_string()),@@ -203,8 +233,7 @@ pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static st}/// Records the resolution of one offloaded model call: the call counter and-/// latency histogram, the `outcome`/`error`/token fields on `span`, and a warn-/// log when the call failed.+/// latency histogram and the outcome/token fields on `span`, without error details.pub(crate) fn record_llm_call(algorithm: &str,selected_model: &str,@@ -230,45 +259,27 @@ pub(crate) fn record_llm_call(.build().record(duration.as_secs_f64() * 1000.0, &call_attributes);- match result {- Ok(response) => {- // Token usage exists only once a response is buffered; a streamed- // response resolves before its usage is known, so none is recorded.- let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {- return;- };- for (field, value) in [- ("input_tokens", usage.input_tokens),- ("output_tokens", usage.output_tokens),- ("total_tokens", usage.total_tokens),- ("reasoning_tokens", usage.reasoning_tokens),- ] {- if let Some(value) = value {- span.record(field, value);- }+ if let Ok(response) = result {+ // Token usage exists only once a response is buffered; a streamed+ // response resolves before its usage is known, so none is recorded.+ let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {+ return;+ };+ for (field, value) in [+ ("input_tokens", usage.input_tokens),+ ("output_tokens", usage.output_tokens),+ ("total_tokens", usage.total_tokens),+ ("reasoning_tokens", usage.reasoning_tokens),+ ] {+ if let Some(value) = value {+ span.record(field, value);}}- Err(error) => {- span.record("error", tracing::field::display(error));- tracing::warn!(- target: TRACING_TARGET,- algorithm,- selected_model,- error = %error,- "model call failed"- );- }}}-/// Records one published routing decision: the decision counter plus a structured debug event.+/// Counts one published routing decision.pub(crate) fn record_decision(algorithm: &str, selected_model: &ModelId) {- tracing::debug!(- target: TRACING_TARGET,- algorithm,- selected_model = %selected_model,- "routing decision"- );meter().u64_counter("switchyard.decisions").build().add(1,&[diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rsindex 756dbc8bf..79adca287 100644--- a/crates/switchyard-py/src/libsy_bindings.rs+++ b/crates/switchyard-py/src/libsy_bindings.rs@@ -514,12 +514,45 @@ impl PyModelCall {}}-/// The terminal routing selection, rewritten request, and optional existing response.+/// Identity and optional JSON evidence from the Rust routing outcome.+#[pyclass(name = "OutcomeMetadata", module = "switchyard.libsy", frozen)]+struct PyOutcomeMetadata {+ inner: switchyard_libsy::OutcomeMetadata,+}++#[pymethods]+impl PyOutcomeMetadata {+ /// UUIDv7 generated for this outcome.+ #[getter]+ fn outcome_id(&self) -> &str {+ self.inner.outcome_id()+ }++ /// Name of the algorithm that produced this outcome.+ #[getter]+ fn algorithm(&self) -> &str {+ &self.inner.algorithm+ }++ /// Optional evidence converted to ordinary Python JSON values.+ #[getter]+ fn evidence(&self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {+ self.inner+ .evidence+ .as_ref()+ .map(|value| to_python(py, value))+ .transpose()+ }+}++/// The terminal routing selection, rewritten request, optional response, and metadata.#[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)]struct PyRoutingOutcome {selected_model_ids: Vec<String>,request: Py<PyAny>,response: Option<Py<PyAny>>,+ #[pyo3(get)]+ metadata: Option<Py<PyOutcomeMetadata>>,}#[pymethods]@@ -675,13 +708,16 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {selected_model_ids,request,response,- metadata: _,+ metadata,} = *outcome;Python::attach(|py| {Ok(PyStep::Done {outcome: Py::new(py,PyRoutingOutcome {+ metadata: metadata+ .map(|inner| Py::new(py, PyOutcomeMetadata { inner }))+ .transpose()?,selected_model_ids: selected_model_ids.iter().map(ToString::to_string)@@ -849,6 +885,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {libsy_module.add_class::<PyLlmResponse>()?;libsy_module.add_class::<PyLlmResponseStream>()?;libsy_module.add_class::<PyModelCall>()?;+ libsy_module.add_class::<PyOutcomeMetadata>()?;libsy_module.add_class::<PyRunStream>()?;libsy_module.add_class::<PyRoutingOutcome>()?;libsy_module.add_class::<PyStep>()?;diff --git a/docs/getting_started.md b/docs/getting_started.mdindex b6070c865..05ec3be0d 100644--- a/docs/getting_started.md+++ b/docs/getting_started.md@@ -213,6 +213,17 @@ response when routing already produced the answer. Otherwise the host makes thecall from that outcome. Serving these calls yourself is what lets libsy embed in a host thatalready owns its HTTP stack, retries, and credentials.+Successful runs also include `OutcomeMetadata`: a unique `outcome_id`, the algorithm+name, and optional JSON evidence. In Python, read `outcome.metadata.outcome_id`,+`outcome.metadata.algorithm`, and `outcome.metadata.evidence` after checking that+`outcome.metadata` is present. Evidence is a normal Python value, usually a dictionary;+algorithms without evidence return `None`.++With a host-installed OpenTelemetry subscriber, the existing `libsy.run` span records+the same identity, selected models, and supported evidence fields. See+the `Algorithm` observability section in the [Rust API reference](reference/rust_api.md)+for the field names. libsy does not install an exporter or send telemetry itself.+For the request, response, and streaming types the steps carry, see[`switchyard-protocol`](../crates/protocol/README.md).diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.pyindex b4817dc87..88a5aa372 100644--- a/switchyard/libsy/__init__.py+++ b/switchyard/libsy/__init__.py@@ -13,6 +13,7 @@ from switchyard_rust.libsy import (LlmFallback,LlmResponse,ModelCall,+ OutcomeMetadata,RoutingOutcome,Step,TaskClassifierConfig,@@ -30,6 +31,7 @@ __all__ = ["LlmFallback","LlmResponse","ModelCall",+ "OutcomeMetadata","RoutingOutcome","Step","TaskClassifierConfig",diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.pyindex 1589d60ff..9756b1c6b 100644--- a/switchyard_rust/libsy.py+++ b/switchyard_rust/libsy.py@@ -21,6 +21,7 @@ _EXPORTS = frozenset("LlmFallback","LlmResponse","ModelCall",+ "OutcomeMetadata","RoutingOutcome","Step","TaskClassifierConfig",@@ -111,8 +112,24 @@ if TYPE_CHECKING:def fail(self, error: BaseException) -> None: ...+ @final+ class OutcomeMetadata:+ """Read-only outcome identity and optional algorithm evidence."""++ @property+ def outcome_id(self) -> str: ...++ @property+ def algorithm(self) -> str: ...++ @property+ def evidence(self) -> Any | None: ...+@finalclass RoutingOutcome:+ @property+ def metadata(self) -> OutcomeMetadata | None: ...+@propertydef selected_model_ids(self) -> list[str]: ...diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.pyindex 02ac614e4..b693d4d87 100644--- a/tests/test_libsy_minimal_bindings.py+++ b/tests/test_libsy_minimal_bindings.py@@ -5,6 +5,7 @@from collections.abc import AsyncIteratorfrom typing import Any+from uuid import UUIDimport pytest@@ -14,6 +15,7 @@ from switchyard.libsy import (CustomClassifierConfig,LlmClassifierConfig,LlmResponse,+ OutcomeMetadata,RoutingOutcome,Step,TaskClassifierConfig,@@ -77,6 +79,8 @@ async def run_algorithm(call.respond(LlmResponse.Agg(response))breakcase Step.Done(outcome):+ assert isinstance(outcome.metadata, OutcomeMetadata)+ assert UUID(outcome.metadata.outcome_id).version == 7if outcome.response is not None:match outcome.response:case LlmResponse.Agg(response):@@ -113,6 +117,10 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()assert outcome is not Noneassert outcome.selected_model_ids == ["fast"]assert outcome.response is None+ assert outcome.metadata is not None+ assert UUID(outcome.metadata.outcome_id).version == 7+ assert outcome.metadata.algorithm == "random"+ assert outcome.metadata.evidence is Noneresponse = await client.call(outcome.request)assert client.calls[0]["model"] == "fast"assert client.calls[0]["messages"][0]["content"] == [@@ -198,14 +206,23 @@ async def test_classifier_config_accepts_a_prompt_override() -> None:),)- _, response = await run_algorithm(- algorithm,- {- "judge": judge,- "weak": weak,- "strong": EchoClient("strong"),- },- )+ outcome: RoutingOutcome | None = None+ async for step in algorithm.run_stream(request_body()):+ match step:+ case Step.CallModel(call):+ call.respond(LlmResponse.Agg(await judge.call(call.request)))+ case Step.Done(done):+ outcome = done++ assert outcome is not None+ assert outcome.selected_model_ids[0] == "weak"+ assert outcome.metadata is not None+ assert outcome.metadata.evidence == {+ "source": "llm-classifier",+ "score": pytest.approx(0.9),+ "threshold": pytest.approx(0.5),+ }+ response = await weak.call(outcome.request)prompt = judge.calls[0]["instructions"][0]["content"][0]["text"]assert prompt == "Custom capability rubric."