Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(libsy): expose outcome metadata in OpenTelemetry and Python (#658)

nachiketb-nvidia578e1b7cbd9d
diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs
index 4d3921afa..8ae0a551e 100644
--- a/crates/libsy-llm-client/tests/observability.rs
+++ b/crates/libsy-llm-client/tests/observability.rs
@@ -488,6 +488,7 @@ impl RoutedLlmClient for UsageClient {
struct SingleCallAlgo {
name: String,
target_set: Vec<String>,
+ metadata: Option<switchyard_libsy::OutcomeMetadata>,
}
#[async_trait]
@@ -507,7 +508,19 @@ impl Algorithm for SingleCallAlgo {
.ok_or(LibsyError::NoTargets)?
.clone();
tracing::info!("picked '{target}'");
- Ok(RoutingOutcome::route_to(target.into(), Vec::new(), request))
+ // Include configured fallbacks so the exported model order can be checked.
+ let mut outcome = RoutingOutcome::route_to(
+ target.into(),
+ self.target_set
+ .iter()
+ .skip(1)
+ .cloned()
+ .map(ModelId::from)
+ .collect(),
+ request,
+ );
+ outcome.metadata = self.metadata.clone();
+ Ok(outcome)
}
}
@@ -561,6 +574,7 @@ fn algo(name: &str, model: &str) -> Arc<dyn Algorithm> {
Arc::new(SingleCallAlgo {
name: name.to_string(),
target_set: vec![model.to_string()],
+ metadata: None,
})
}
@@ -748,7 +762,8 @@ async fn affinity_keeps_the_algorithm_selection_after_client_fallback()
}
#[tokio::test]
-async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_libsy::Result<()> {
+async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchyard_libsy::Result<()>
+{
let _guard = serialize_test().lock().await;
let (store, exporter, provider, span_exporter, _) = telemetry();
const ALGO: &str = "obs-success-algo";
@@ -775,7 +790,20 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
request.llm_request.output.max_output_tokens = Some(512);
request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));
request.llm_request.reasoning.effort = Some("high".to_string());
- let (selected_model, _response) = run(algo(ALGO, MODEL), client, request).await?;
+ let metadata = switchyard_libsy::OutcomeMetadata::new(
+ ALGO.to_string(),
+ Some(json!({
+ "source": "llm-classifier", "score": 0.9, "threshold": 0.5,
+ "verdict": "continue", "trigger": "turn", "reason_code": "test",
+ "confidence": "wrong type", "unknown": LEAKED_CONTENT,
+ })),
+ );
+ let algorithm = Arc::new(SingleCallAlgo {
+ name: ALGO.to_string(),
+ target_set: vec![MODEL.to_string(), "obs-fallback-model".to_string()],
+ metadata: Some(metadata.clone()),
+ });
+ let (selected_model, _response) = run(algorithm, client, request).await?;
assert_eq!(selected_model, MODEL);
// Metrics: run/call counters and latency histograms keyed by algorithm,
@@ -843,6 +871,51 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
// llm_call span carrying the selection, outcome, and token counts.
let spans = store.spans();
let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO);
+ assert_eq!(
+ spans
+ .iter()
+ .filter(|span| span.name == "libsy.run"
+ && span.fields.get("algorithm").map(String::as_str) == Some(ALGO))
+ .count(),
+ 1
+ );
+ for (field, expected) in [
+ ("outcome_id", metadata.outcome_id()),
+ ("evidence.source", "llm-classifier"),
+ ("evidence.score", "0.9"),
+ ("evidence.threshold", "0.5"),
+ ("evidence.verdict", "continue"),
+ ("evidence.trigger", "turn"),
+ ("evidence.reason_code", "test"),
+ ] {
+ assert_eq!(
+ run_span.fields.get(field).map(String::as_str),
+ Some(expected),
+ "{field}"
+ );
+ }
+ assert!(!run_span.fields.contains_key("evidence.confidence"));
+ assert!(!format!("{run_span:?}").contains(LEAKED_CONTENT));
+ let exported = span_exporter.get_finished_spans().expect("exported spans");
+ let exported_run = exported
+ .iter()
+ .find(|span| {
+ span.name == "libsy.run"
+ && otel_attribute(span, "outcome_id")
+ .is_some_and(|id| id.as_str() == metadata.outcome_id())
+ })
+ .expect("outcome span exported");
+ assert_eq!(
+ otel_attribute(exported_run, "evidence.score"),
+ Some(&OtelValue::F64(0.9))
+ );
+ assert_eq!(
+ otel_attribute(exported_run, "selected_model_ids"),
+ Some(&OtelValue::Array(OtelArray::String(vec![
+ MODEL.into(),
+ "obs-fallback-model".into()
+ ]),))
+ );
assert_eq!(run_span.parent, None);
assert_eq!(
run_span.fields.get("session_id").map(String::as_str),
@@ -880,13 +953,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
run_span.fields.get("agent_role").map(String::as_str),
Some("reviewer")
);
- // Host-defined labels ride in generically via Metadata.extra_metadata.
- assert!(
- run_span
- .fields
- .get("extra_metadata")
- .is_some_and(|extra| extra.contains("tenant") && extra.contains("obs-tenant-1"))
- );
+ assert!(!run_span.fields.contains_key("extra_metadata"));
// The default-client serve inside `run` gets its own client-call span.
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
@@ -950,6 +1017,12 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
// The algorithm logs why it made the decision.
let events = store.events();
+ assert!(
+ !events.iter().any(
+ |event| event.fields.get("algorithm").map(String::as_str) == Some(ALGO)
+ && event.fields.get("message").map(String::as_str) == Some("routing decision")
+ )
+ );
assert!(
events.iter().any(|event| {
event.level == "INFO"
@@ -1250,7 +1323,7 @@ async fn upstream_body_is_redacted_from_the_client_call_span() -> switchyard_lib
}
#[tokio::test]
-async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Result<()> {
+async fn failed_call_records_metrics_without_error_details() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, exporter, provider, _, _) = telemetry();
const ALGO: &str = "obs-failure-algo";
@@ -1272,7 +1345,10 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
Ok(Step::Done(_)) => {
return Err(test_error("expected the failed call to fail the run"));
}
- Err(_) => saw_error_step = true,
+ Err(error) => {
+ assert!(error.to_string().contains("synthetic upstream failure"));
+ saw_error_step = true;
+ }
}
}
assert!(
@@ -1307,51 +1383,27 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
None
);
- // Spans: both spans carry outcome=error and the propagated error text.
+ // Error details remain with the caller, while operational status is preserved.
let spans = store.spans();
let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO);
assert_eq!(
run_span.fields.get("outcome").map(String::as_str),
Some("error")
);
- assert!(
- run_span
- .fields
- .get("error")
- .is_some_and(|error| error.contains("synthetic upstream failure"))
- );
+ assert!(!run_span.fields.contains_key("error"));
+ assert!(!run_span.fields.contains_key("outcome_id"));
let call_span = find_span(&spans, "libsy.llm_call", "selected_model", MODEL);
assert_eq!(
call_span.fields.get("outcome").map(String::as_str),
Some("error")
);
- // Structured logs warn once for the failed call and failed run.
+ assert_eq!(call_span.parent.as_deref(), Some("libsy.run"));
+ assert!(!call_span.fields.contains_key("error"));
+
let events = store.events();
- assert!(
- events.iter().any(|event| {
- event.target == "libsy"
- && event.level == "WARN"
- && event.fields.get("selected_model").map(String::as_str) == Some(MODEL)
- && event
- .fields
- .get("message")
- .is_some_and(|message| message.contains("model call failed"))
- }),
- "no call-failure log for {MODEL} in {events:?}"
- );
- assert!(
- events.iter().any(|event| {
- event.target == "libsy"
- && event.level == "WARN"
- && event.fields.get("algorithm").map(String::as_str) == Some(ALGO)
- && event
- .fields
- .get("message")
- .is_some_and(|message| message.contains("algorithm run failed"))
- }),
- "no run-failure log for {ALGO} in {events:?}"
- );
+ assert!(!events.iter().any(|event| event.target == "libsy"
+ && event.fields.get("algorithm").map(String::as_str) == Some(ALGO)));
Ok(())
}
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index cecab86af..6e01c20c7 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -176,7 +176,6 @@ impl Driver {
selected_model = %models.first().map(ModelId::as_str).unwrap_or("NoTargets"),
openinference.span.kind = "CHAIN",
outcome = tracing::field::Empty,
- error = tracing::field::Empty,
input_tokens = tracing::field::Empty,
output_tokens = tracing::field::Empty,
total_tokens = tracing::field::Empty,
@@ -225,7 +224,7 @@ impl Driver {
let metadata = outcome.metadata.get_or_insert_with(|| {
crate::OutcomeMetadata::new(self.algorithm.clone(), self.evidence.lock().take())
});
- tracing::Span::current().record("outcome_id", metadata.outcome_id());
+ observability::record_outcome(metadata, &outcome.selected_model_ids);
outcome
});
let selected_model = result
@@ -383,9 +382,20 @@ impl RoutingIdentity {
/// # Observability
///
/// [`run_stream`](Self::run_stream) creates a `libsy.run` span, and each offloaded model
-/// call creates a `libsy.llm_call` span. Routing decisions and failures are emitted through
-/// `tracing`; metrics use the global OpenTelemetry meter provider. The provider call
-/// itself belongs to the host, and is instrumented by whoever makes it.
+/// call creates a nested `libsy.llm_call` span. Successful outcomes record their
+/// [`OutcomeMetadata::outcome_id`](crate::OutcomeMetadata::outcome_id) on `libsy.run`,
+/// alongside `selected_model_ids` (an ordered OpenTelemetry string array).
+/// `algorithm` and `switchyard.algorithm` retain the run's [`Algorithm::name`].
+/// Optional `evidence.source`, `evidence.verdict`, `evidence.trigger`, and
+/// `evidence.reason_code` are strings; `evidence.score`, `evidence.confidence`, and
+/// `evidence.threshold` are numbers. Unknown evidence fields are not exported.
+/// These fields are span attributes, never metric labels.
+///
+/// The run/call observability helpers retain `outcome` status and operational metrics,
+/// but omit error details and arbitrary request extra metadata. Algorithms and hosts
+/// may emit their own logs. Errors still reach the caller unchanged.
+/// The host controls the tracing subscriber and global OpenTelemetry
+/// meter provider; libsy installs no exporter and performs no telemetry network I/O.
#[async_trait]
pub trait Algorithm: Send + Sync + 'static {
/// Stable, low-cardinality name identifying this algorithm — the
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 763d31153..69c6d9295 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -38,8 +38,9 @@ use std::time::{Duration, Instant};
use opentelemetry::metrics::Meter;
use opentelemetry::{KeyValue, global};
use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
-use crate::Result;
+use crate::{OutcomeMetadata, Result};
use switchyard_protocol::{ModelId, Request, Response};
const METRICS_SCOPE: &str = "switchyard";
@@ -61,16 +62,22 @@ pub(crate) fn outcome_value<T>(result: &Result<T>) -> &'static str {
/// Span covering one algorithm run (the whole `route` execution).
///
/// Correlation ids from the request [`switchyard_protocol::Metadata`] are recorded as span fields
-/// when present. `tracing` spans cannot grow field names at runtime, so
-/// arbitrary host labels ride in via [`switchyard_protocol::Metadata::extra_metadata`], recorded
-/// whole into the `extra_metadata` field. `outcome` and `error` are filled in
-/// by [`record_run`] when the run ends.
+/// when present. Arbitrary extra metadata and error details are not exported.
+/// [`record_outcome`] fills in successful outcome fields; [`record_run`] records
+/// whether the run succeeded.
pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {
let span = tracing::info_span!(
target: TRACING_TARGET,
"libsy.run",
algorithm,
outcome_id = tracing::field::Empty,
+ evidence.source = tracing::field::Empty,
+ evidence.score = tracing::field::Empty,
+ evidence.confidence = tracing::field::Empty,
+ evidence.threshold = tracing::field::Empty,
+ evidence.verdict = tracing::field::Empty,
+ evidence.trigger = tracing::field::Empty,
+ evidence.reason_code = tracing::field::Empty,
switchyard.algorithm = algorithm,
openinference.span.kind = "CHAIN",
switchyard.route = tracing::field::Empty,
@@ -81,9 +88,7 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {
task_kind = tracing::field::Empty,
agent_role = tracing::field::Empty,
correlation_id = tracing::field::Empty,
- extra_metadata = tracing::field::Empty,
outcome = tracing::field::Empty,
- error = tracing::field::Empty,
);
if let Some(route) = request.model_id() {
span.record("switchyard.route", route.as_ref());
@@ -104,13 +109,48 @@ pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {
if let Some(session_id) = &metadata.session_id {
span.record("session.id", session_id.as_str());
}
- if let Some(extra) = &metadata.extra_metadata {
- span.record("extra_metadata", tracing::field::debug(extra));
- }
}
span
}
+/// Projects a successful outcome onto the existing run span. Model IDs are an
+/// ordered OpenTelemetry string array, preserving fallback order. Evidence uses typed fields;
+/// unknown keys and values of the wrong type are omitted.
+pub(crate) fn record_outcome(metadata: &OutcomeMetadata, models: &[ModelId]) {
+ let span = Span::current();
+ span.record("outcome_id", metadata.outcome_id());
+ span.set_attribute(
+ "selected_model_ids",
+ opentelemetry::Value::Array(opentelemetry::Array::String(
+ models
+ .iter()
+ .map(|model| model.to_string().into())
+ .collect(),
+ )),
+ );
+ if let Some(evidence) = &metadata.evidence {
+ for (key, field) in [
+ ("source", "evidence.source"),
+ ("verdict", "evidence.verdict"),
+ ("trigger", "evidence.trigger"),
+ ("reason_code", "evidence.reason_code"),
+ ] {
+ if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_str) {
+ span.record(field, value);
+ }
+ }
+ for (key, field) in [
+ ("score", "evidence.score"),
+ ("confidence", "evidence.confidence"),
+ ("threshold", "evidence.threshold"),
+ ] {
+ if let Some(value) = evidence.get(key).and_then(serde_json::Value::as_f64) {
+ span.record(field, value);
+ }
+ }
+ }
+}
+
/// Holds `switchyard.algorithms_in_flight` up by one for as long as it lives.
struct InFlightRun {
algorithm: String,
@@ -141,7 +181,7 @@ fn record_algorithms_in_flight(algorithm: &str, delta: i64) {
}
/// Runs one algorithm task to completion, recording the run counter, duration
-/// histogram, span outcome, and failure log when it resolves. Counts the run as
+/// histogram and span outcome when it resolves. Counts the run as
/// in flight for its whole duration.
/// Executes inside the `libsy.run` span its caller instruments the task with.
pub(crate) async fn observe_run<T>(
@@ -158,20 +198,10 @@ pub(crate) async fn observe_run<T>(
}
/// Records the end of one algorithm run: the run counter and duration
-/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the
-/// run failed.
+/// histogram and the `outcome` field on `span`, without error details.
fn record_run<T>(algorithm: &str, duration: Duration, result: &Result<T>, span: &Span) {
let outcome = outcome_value(result);
span.record("outcome", outcome);
- if let Err(error) = result {
- span.record("error", tracing::field::display(error));
- tracing::warn!(
- target: TRACING_TARGET,
- algorithm,
- error = %error,
- "algorithm run failed"
- );
- }
let attributes = [
KeyValue::new("algorithm", algorithm.to_string()),
@@ -203,8 +233,7 @@ pub(crate) fn record_classifier_fail_open(judge_model: &str, reason: &'static st
}
/// Records the resolution of one offloaded model call: the call counter and
-/// latency histogram, the `outcome`/`error`/token fields on `span`, and a warn
-/// log when the call failed.
+/// latency histogram and the outcome/token fields on `span`, without error details.
pub(crate) fn record_llm_call(
algorithm: &str,
selected_model: &str,
@@ -230,45 +259,27 @@ pub(crate) fn record_llm_call(
.build()
.record(duration.as_secs_f64() * 1000.0, &call_attributes);
- match result {
- Ok(response) => {
- // Token usage exists only once a response is buffered; a streamed
- // response resolves before its usage is known, so none is recorded.
- let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {
- return;
- };
- for (field, value) in [
- ("input_tokens", usage.input_tokens),
- ("output_tokens", usage.output_tokens),
- ("total_tokens", usage.total_tokens),
- ("reasoning_tokens", usage.reasoning_tokens),
- ] {
- if let Some(value) = value {
- span.record(field, value);
- }
+ if let Ok(response) = result {
+ // Token usage exists only once a response is buffered; a streamed
+ // response resolves before its usage is known, so none is recorded.
+ let Some(usage) = response.llm_response.as_agg().map(|agg| &agg.usage) else {
+ return;
+ };
+ for (field, value) in [
+ ("input_tokens", usage.input_tokens),
+ ("output_tokens", usage.output_tokens),
+ ("total_tokens", usage.total_tokens),
+ ("reasoning_tokens", usage.reasoning_tokens),
+ ] {
+ if let Some(value) = value {
+ span.record(field, value);
}
}
- Err(error) => {
- span.record("error", tracing::field::display(error));
- tracing::warn!(
- target: TRACING_TARGET,
- algorithm,
- selected_model,
- error = %error,
- "model call failed"
- );
- }
}
}
-/// Records one published routing decision: the decision counter plus a structured debug event.
+/// Counts one published routing decision.
pub(crate) fn record_decision(algorithm: &str, selected_model: &ModelId) {
- tracing::debug!(
- target: TRACING_TARGET,
- algorithm,
- selected_model = %selected_model,
- "routing decision"
- );
meter().u64_counter("switchyard.decisions").build().add(
1,
&[
diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs
index 756dbc8bf..79adca287 100644
--- a/crates/switchyard-py/src/libsy_bindings.rs
+++ b/crates/switchyard-py/src/libsy_bindings.rs
@@ -514,12 +514,45 @@ impl PyModelCall {
}
}
-/// The terminal routing selection, rewritten request, and optional existing response.
+/// Identity and optional JSON evidence from the Rust routing outcome.
+#[pyclass(name = "OutcomeMetadata", module = "switchyard.libsy", frozen)]
+struct PyOutcomeMetadata {
+ inner: switchyard_libsy::OutcomeMetadata,
+}
+
+#[pymethods]
+impl PyOutcomeMetadata {
+ /// UUIDv7 generated for this outcome.
+ #[getter]
+ fn outcome_id(&self) -> &str {
+ self.inner.outcome_id()
+ }
+
+ /// Name of the algorithm that produced this outcome.
+ #[getter]
+ fn algorithm(&self) -> &str {
+ &self.inner.algorithm
+ }
+
+ /// Optional evidence converted to ordinary Python JSON values.
+ #[getter]
+ fn evidence(&self, py: Python<'_>) -> PyResult<Option<Py<PyAny>>> {
+ self.inner
+ .evidence
+ .as_ref()
+ .map(|value| to_python(py, value))
+ .transpose()
+ }
+}
+
+/// The terminal routing selection, rewritten request, optional response, and metadata.
#[pyclass(name = "RoutingOutcome", module = "switchyard.libsy", frozen)]
struct PyRoutingOutcome {
selected_model_ids: Vec<String>,
request: Py<PyAny>,
response: Option<Py<PyAny>>,
+ #[pyo3(get)]
+ metadata: Option<Py<PyOutcomeMetadata>>,
}
#[pymethods]
@@ -675,13 +708,16 @@ fn step_to_python(step: RustStep) -> PyResult<PyStep> {
selected_model_ids,
request,
response,
- metadata: _,
+ metadata,
} = *outcome;
Python::attach(|py| {
Ok(PyStep::Done {
outcome: Py::new(
py,
PyRoutingOutcome {
+ metadata: metadata
+ .map(|inner| Py::new(py, PyOutcomeMetadata { inner }))
+ .transpose()?,
selected_model_ids: selected_model_ids
.iter()
.map(ToString::to_string)
@@ -849,6 +885,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
libsy_module.add_class::<PyLlmResponse>()?;
libsy_module.add_class::<PyLlmResponseStream>()?;
libsy_module.add_class::<PyModelCall>()?;
+ libsy_module.add_class::<PyOutcomeMetadata>()?;
libsy_module.add_class::<PyRunStream>()?;
libsy_module.add_class::<PyRoutingOutcome>()?;
libsy_module.add_class::<PyStep>()?;
diff --git a/docs/getting_started.md b/docs/getting_started.md
index b6070c865..05ec3be0d 100644
--- a/docs/getting_started.md
+++ b/docs/getting_started.md
@@ -213,6 +213,17 @@ response when routing already produced the answer. Otherwise the host makes the
call from that outcome. Serving these calls yourself is what lets libsy embed in a host that
already owns its HTTP stack, retries, and credentials.
+Successful runs also include `OutcomeMetadata`: a unique `outcome_id`, the algorithm
+name, and optional JSON evidence. In Python, read `outcome.metadata.outcome_id`,
+`outcome.metadata.algorithm`, and `outcome.metadata.evidence` after checking that
+`outcome.metadata` is present. Evidence is a normal Python value, usually a dictionary;
+algorithms without evidence return `None`.
+
+With a host-installed OpenTelemetry subscriber, the existing `libsy.run` span records
+the same identity, selected models, and supported evidence fields. See
+the `Algorithm` observability section in the [Rust API reference](reference/rust_api.md)
+for the field names. libsy does not install an exporter or send telemetry itself.
+
For the request, response, and streaming types the steps carry, see
[`switchyard-protocol`](../crates/protocol/README.md).
diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py
index b4817dc87..88a5aa372 100644
--- a/switchyard/libsy/__init__.py
+++ b/switchyard/libsy/__init__.py
@@ -13,6 +13,7 @@ from switchyard_rust.libsy import (
LlmFallback,
LlmResponse,
ModelCall,
+ OutcomeMetadata,
RoutingOutcome,
Step,
TaskClassifierConfig,
@@ -30,6 +31,7 @@ __all__ = [
"LlmFallback",
"LlmResponse",
"ModelCall",
+ "OutcomeMetadata",
"RoutingOutcome",
"Step",
"TaskClassifierConfig",
diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py
index 1589d60ff..9756b1c6b 100644
--- a/switchyard_rust/libsy.py
+++ b/switchyard_rust/libsy.py
@@ -21,6 +21,7 @@ _EXPORTS = frozenset(
"LlmFallback",
"LlmResponse",
"ModelCall",
+ "OutcomeMetadata",
"RoutingOutcome",
"Step",
"TaskClassifierConfig",
@@ -111,8 +112,24 @@ if TYPE_CHECKING:
def fail(self, error: BaseException) -> None: ...
+ @final
+ class OutcomeMetadata:
+ """Read-only outcome identity and optional algorithm evidence."""
+
+ @property
+ def outcome_id(self) -> str: ...
+
+ @property
+ def algorithm(self) -> str: ...
+
+ @property
+ def evidence(self) -> Any | None: ...
+
@final
class RoutingOutcome:
+ @property
+ def metadata(self) -> OutcomeMetadata | None: ...
+
@property
def selected_model_ids(self) -> list[str]: ...
diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py
index 02ac614e4..b693d4d87 100644
--- a/tests/test_libsy_minimal_bindings.py
+++ b/tests/test_libsy_minimal_bindings.py
@@ -5,6 +5,7 @@
from collections.abc import AsyncIterator
from typing import Any
+from uuid import UUID
import pytest
@@ -14,6 +15,7 @@ from switchyard.libsy import (
CustomClassifierConfig,
LlmClassifierConfig,
LlmResponse,
+ OutcomeMetadata,
RoutingOutcome,
Step,
TaskClassifierConfig,
@@ -77,6 +79,8 @@ async def run_algorithm(
call.respond(LlmResponse.Agg(response))
break
case Step.Done(outcome):
+ assert isinstance(outcome.metadata, OutcomeMetadata)
+ assert UUID(outcome.metadata.outcome_id).version == 7
if outcome.response is not None:
match outcome.response:
case LlmResponse.Agg(response):
@@ -113,6 +117,10 @@ async def test_random_streams_complex_steps_and_accepts_a_dictionary_response()
assert outcome is not None
assert outcome.selected_model_ids == ["fast"]
assert outcome.response is None
+ assert outcome.metadata is not None
+ assert UUID(outcome.metadata.outcome_id).version == 7
+ assert outcome.metadata.algorithm == "random"
+ assert outcome.metadata.evidence is None
response = await client.call(outcome.request)
assert client.calls[0]["model"] == "fast"
assert client.calls[0]["messages"][0]["content"] == [
@@ -198,14 +206,23 @@ async def test_classifier_config_accepts_a_prompt_override() -> None:
),
)
- _, response = await run_algorithm(
- algorithm,
- {
- "judge": judge,
- "weak": weak,
- "strong": EchoClient("strong"),
- },
- )
+ outcome: RoutingOutcome | None = None
+ async for step in algorithm.run_stream(request_body()):
+ match step:
+ case Step.CallModel(call):
+ call.respond(LlmResponse.Agg(await judge.call(call.request)))
+ case Step.Done(done):
+ outcome = done
+
+ assert outcome is not None
+ assert outcome.selected_model_ids[0] == "weak"
+ assert outcome.metadata is not None
+ assert outcome.metadata.evidence == {
+ "source": "llm-classifier",
+ "score": pytest.approx(0.9),
+ "threshold": pytest.approx(0.5),
+ }
+ response = await weak.call(outcome.request)
prompt = judge.calls[0]["instructions"][0]["content"][0]["text"]
assert prompt == "Custom capability rubric."