Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(relay): capture typed decision payloads (#1254)

Alex Fourniere412371c969b
diff --git a/crates/libsy-llm-client/src/openai_decisions.rs b/crates/libsy-llm-client/src/openai_decisions.rs
index b6f93edb7..5a6ce2a79 100644
--- a/crates/libsy-llm-client/src/openai_decisions.rs
+++ b/crates/libsy-llm-client/src/openai_decisions.rs
@@ -51,15 +51,21 @@ impl OpenAiDecisionClient {
#[async_trait]
impl RoutedDecisionClient for OpenAiDecisionClient {
async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {
- let response = self
+ let builder = self
.client
.post(self.endpoint.clone())
.bearer_auth(&self.api_key)
- .json(&encode(&request)?)
- .send()
- .await
- .inspect_err(|_| metrics::record_upstream_attempt(None))
- .map_err(convert_reqwest_error)?;
+ .json(&encode(&request)?);
+ let response = async {
+ let request = builder.build()?;
+ if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) {
+ crate::payload_capture::record("request", || String::from_utf8_lossy(body));
+ }
+ self.client.execute(request).await
+ }
+ .await
+ .inspect_err(|_| metrics::record_upstream_attempt(None))
+ .map_err(convert_reqwest_error)?;
let status = response.status();
let body = response
.bytes()
@@ -73,6 +79,7 @@ impl RoutedDecisionClient for OpenAiDecisionClient {
body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),
});
}
+ crate::payload_capture::record("response", || String::from_utf8_lossy(&body));
let response: WireResponse =
serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {
source: Box::new(source),
diff --git a/crates/libsy-llm-client/src/system_one.rs b/crates/libsy-llm-client/src/system_one.rs
index cc2b40011..e0b22e19d 100644
--- a/crates/libsy-llm-client/src/system_one.rs
+++ b/crates/libsy-llm-client/src/system_one.rs
@@ -51,15 +51,21 @@ impl SystemOneClient {
#[async_trait]
impl RoutedDecisionClient for SystemOneClient {
async fn call(&self, request: DecisionRequest) -> Result<DecisionResponse> {
- let response = self
+ let builder = self
.client
.post(self.endpoint.clone())
.bearer_auth(&self.api_key)
- .json(&encode(&request)?)
- .send()
- .await
- .inspect_err(|_| metrics::record_upstream_attempt(None))
- .map_err(convert_reqwest_error)?;
+ .json(&encode(&request)?);
+ let response = async {
+ let request = builder.build()?;
+ if let Some(body) = request.body().and_then(reqwest::Body::as_bytes) {
+ crate::payload_capture::record("request", || String::from_utf8_lossy(body));
+ }
+ self.client.execute(request).await
+ }
+ .await
+ .inspect_err(|_| metrics::record_upstream_attempt(None))
+ .map_err(convert_reqwest_error)?;
let status = response.status();
let body = response
.bytes()
@@ -73,6 +79,7 @@ impl RoutedDecisionClient for SystemOneClient {
body: String::from_utf8_lossy(&body).replace(&self.api_key, "[REDACTED]"),
});
}
+ crate::payload_capture::record("response", || String::from_utf8_lossy(&body));
let response: WireResponse =
serde_json::from_slice(&body).map_err(|source| LlmClientError::InvalidResponse {
source: Box::new(source),
diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md
index 781c04e6d..4b5819dbe 100644
--- a/crates/switchyard-nemo-relay-plugin/README.md
+++ b/crates/switchyard-nemo-relay-plugin/README.md
@@ -231,11 +231,12 @@ marks. Call marks distinguish routing from answer calls; decisions distinguish
selected from served models. Token metrics cover both call roles, while Relay
retains ownership of the outer LLM lifecycle.
-The plugin also captures the existing `libsy.client_call` and
-`libsy.upstream_attempt` spans for each routed request. It replays completed
+The plugin also captures the existing `libsy.client_call`,
+`libsy.upstream_attempt`, and `libsy.decision_client_call` spans for each routed
+request. It replays completed
spans beneath Relay's outer LLM scope with their original timestamps. The
projection contains allowlisted routing metadata by default. To inspect the
-final classifier/judge request and buffered response, enable
+final classifier/judge or typed decision request and buffered response, enable
[routing payload capture](../../docs/integrations/nemo_relay.md#inspect-classifier-and-judge-payloads)
in the plugin configuration. Capture uses Relay's existing exporters and writes
no payload files inside the plugin. It is disabled by default and omits bodies
diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
index f9886b005..e1dd53a46 100644
--- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
+++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
@@ -22,6 +22,7 @@ use tracing_subscriber::registry::LookupSpan;
const CLIENT_CALL: &str = "libsy.client_call";
const UPSTREAM_ATTEMPT: &str = "libsy.upstream_attempt";
+const DECISION_CALL: &str = "libsy.decision_client_call";
const PAYLOAD_TARGET: &str = "switchyard.payload";
const MAX_PAYLOAD_REDACTION_DEPTH: usize = 64;
@@ -71,17 +72,18 @@ where
let Some(span) = ctx.event_span(event) else {
return;
};
- let is_routing = span.scope().any(|ancestor| {
- ancestor.extensions().get::<OpenSpan>().is_some_and(|open| {
- open.name == CLIENT_CALL
- && open
- .fields
- .get("switchyard.call_phase")
- .and_then(Value::as_str)
- == Some("routing")
- })
- });
- if !is_routing || span.name() != UPSTREAM_ATTEMPT {
+ let is_routing = span.name() == DECISION_CALL
+ || span.scope().any(|ancestor| {
+ ancestor.extensions().get::<OpenSpan>().is_some_and(|open| {
+ open.name == CLIENT_CALL
+ && open
+ .fields
+ .get("switchyard.call_phase")
+ .and_then(Value::as_str)
+ == Some("routing")
+ })
+ });
+ if !is_routing || !matches!(span.name(), UPSTREAM_ATTEMPT | DECISION_CALL) {
return;
}
let mut payload = PayloadVisitor {
@@ -510,7 +512,8 @@ impl Visit for FieldVisitor<'_> {
}
fn captures(metadata: &Metadata<'_>) -> bool {
- metadata.target() == "libsy" && matches!(metadata.name(), CLIENT_CALL | UPSTREAM_ATTEMPT)
+ (metadata.target() == "libsy" && matches!(metadata.name(), CLIENT_CALL | UPSTREAM_ATTEMPT))
+ || (metadata.target() == "switchyard_llm_client::run" && metadata.name() == DECISION_CALL)
}
fn relay_scope_type(name: &str) -> ScopeType {
@@ -708,6 +711,115 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn captures_decision_http_bodies_with_opt_in_and_size_limits() {
+ use std::{collections::BTreeMap, sync::Arc, time::Duration};
+
+ use switchyard_llm_client::{ClientRouter, OpenAiDecisionClient, SystemOneClient, decide};
+ use switchyard_protocol::{Request, RoutedDecisionClient, text_request};
+ use switchyard_runner::AlgorithmSpec;
+ use wiremock::matchers::{header, method};
+ use wiremock::{Mock, MockServer, ResponseTemplate};
+
+ let spec: AlgorithmSpec = serde_json::from_value(serde_json::json!({
+ "type": "llm_classifier", "classifier_target": "decision/model",
+ "strong_target": "capable", "weak_target": "efficient",
+ "decision": {"instructions": "Choose capable or efficient."}, "fail_open": false
+ }))
+ .unwrap();
+ let targets = BTreeMap::from(
+ ["decision/model", "capable", "efficient"].map(|model| (model.into(), model.into())),
+ );
+ let (algorithm, models) = spec.build_with_runtime_models("capture", &targets).unwrap();
+ let models = Arc::new(models);
+ let redactor = ProviderKeyRedactor::new(&["qa-secret".into()]);
+ for system_one in [true, false] {
+ let server = MockServer::start().await;
+ let response = if system_one {
+ serde_json::json!({
+ "model": "decision/model", "answers": {"route": {
+ "type": "choice", "choice": "capable",
+ "probabilities": {"capable": 0.9, "efficient": 0.1}}}
+ })
+ } else {
+ serde_json::json!({
+ "model": "decision/model", "answers": [{"name": "route", "type": "choice",
+ "choice": "capable", "probabilities": [
+ {"value": "capable", "probability": 0.9},
+ {"value": "efficient", "probability": 0.1}]}],
+ "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2,
+ "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0},
+ "output_tokens_details": {"reasoning_tokens": 0}}
+ })
+ };
+ Mock::given(method("POST"))
+ .and(header("authorization", "Bearer qa-secret"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(&response))
+ .expect(3)
+ .mount(&server)
+ .await;
+ let endpoint = server.uri().parse().unwrap();
+ let client: Arc<dyn RoutedDecisionClient> = if system_one {
+ Arc::new(
+ SystemOneClient::new(endpoint, "qa-secret".into(), Duration::from_secs(5))
+ .unwrap(),
+ )
+ } else {
+ Arc::new(
+ OpenAiDecisionClient::new(endpoint, "qa-secret".into(), Duration::from_secs(5))
+ .unwrap(),
+ )
+ };
+ let clients = ClientRouter::new_with_decision_clients(
+ Default::default(),
+ [("decision/model".into(), client)].into(),
+ Default::default(),
+ None,
+ &[],
+ );
+ for limit in [None, Some(4096), Some(1)] {
+ let (sender, spans) = mpsc::channel();
+ let capture = SpanCapture::new(limit, move |span| sender.send(span).unwrap());
+ let request = Request {
+ llm_request: text_request(None, "Inspect qa-secret."),
+ ..Request::default()
+ };
+ let execution = decide(algorithm.clone(), clients.clone(), request, models.clone());
+ capture.capture(execution).await.unwrap();
+ let mut span = spans.try_recv().expect("decision span must be captured");
+ assert_eq!(span.name, DECISION_CALL);
+ assert_eq!(span.fields["outcome"], "ok");
+ assert!(spans.try_recv().is_err());
+ let received = server.received_requests().await.unwrap();
+ let request_body = received.last().unwrap().body_json::<Json>().unwrap();
+ for (payload, expected) in [
+ (&span.payloads.request, &request_body),
+ (&span.payloads.response, &response),
+ ] {
+ match limit {
+ None => assert!(payload.is_none()),
+ Some(1) => assert_eq!(
+ payload.as_ref().unwrap(),
+ &serde_json::json!({"body": null, "truncated": true})
+ ),
+ Some(_) => {
+ let body = payload.as_ref().unwrap()["body"].as_str().unwrap();
+ assert_eq!(&serde_json::from_str::<Json>(body).unwrap(), expected);
+ }
+ }
+ }
+ span.sanitize(&redactor);
+ if limit == Some(4096) {
+ let payload = span.payloads.request.unwrap();
+ let body = payload["body"].as_str().unwrap();
+ assert!(body.contains("Inspect") && body.contains("[REDACTED]"));
+ assert!(!body.contains("qa-secret"));
+ }
+ }
+ server.verify().await;
+ }
+ }
+
#[test]
fn captured_classifier_content_and_tool_arguments_redact_nested_json() {
let nested = r#"{"credential":"qa-\u0073ecret","qa-\u0073ecret":"ordinary"}"#;
diff --git a/docs/integrations/nemo_relay.md b/docs/integrations/nemo_relay.md
index 7c385607e..ea2955970 100644
--- a/docs/integrations/nemo_relay.md
+++ b/docs/integrations/nemo_relay.md
@@ -305,14 +305,15 @@ can receive these records, but each output format presents them differently.
| Output | What the current integration shows |
| --- | --- |
| ATIF | The request and response seen by the agent. Internal routing is not added as separate steps or included in `final_metrics`. Relay-managed ATIF files may retain the raw records under `extra.observed_events`. |
-| OpenTelemetry traces, including OpenInference | Relay's outer LLM span, nested `libsy.client_call` and `libsy.upstream_attempt` duration scopes, plus eligible routing marks. |
+| OpenTelemetry traces, including OpenInference | Relay's outer LLM span, nested `libsy.client_call`, `libsy.upstream_attempt`, and `libsy.decision_client_call` duration scopes, plus eligible routing marks. |
| OpenTelemetry metrics | Switchyard request, routing-call, latency, token, and failure measurements. Combine the reported routing usage with Relay's answer cost to estimate the observed routed cost. |
| OTLP logs | Non-metric Switchyard marks that meet the configured severity threshold. |
-The plugin captures two existing Switchyard spans. Each routing or answer LLM
+The plugin captures existing Switchyard spans. Each routing or answer LLM
call becomes a `libsy.client_call` scope. Physical HTTP attempts and retries
become `libsy.upstream_attempt` child scopes. Buffered calls close with the response;
-streaming calls close when the stream finishes, fails, or is dropped. The
+streaming calls close when the stream finishes, fails, or is dropped. Typed
+decision calls become `libsy.decision_client_call` scopes. The
projection contains allowlisted routing metadata by default. Optional
[routing payload capture](#inspect-classifier-and-judge-payloads) adds the final
classifier/judge request and successful buffered response.
@@ -328,6 +329,7 @@ classification as `routing` or `answer`.
| --- | --- | --- | --- |
| Routing or answer model call | `libsy.client_call` scope | Relay's managed request scope | Starts with the Switchyard call and closes with the buffered response or terminal stream event. |
| Physical provider request or retry | `libsy.upstream_attempt` scope | Its `libsy.client_call` scope | Starts before the HTTP attempt and closes on response, error, or cancellation. |
+| System One or OpenAI Decisions call | `libsy.decision_client_call` scope | Relay's managed request scope | Starts with the decision call and closes on response, error, or cancellation. |
| Routing request, decision, overhead, or failure | Existing `switchyard.routing.*` mark | Active Relay scope at emission | Point-in-time mark using the existing outcome fields. |
### Inspect Classifier and Judge Payloads
@@ -337,7 +339,8 @@ is disabled by default. It is not a standalone Switchyard configuration option.
Use it to inspect the final classifier/judge request, including its system
prompt, task context, output schema, resolved model, and target body settings,
plus its successful buffered response. This helps distinguish a poor verdict
-from missing task context.
+from missing task context. System One and OpenAI Decisions requests and
+successful buffered responses are supported too.
Add these options to the existing `plugins.dynamic.config` table in your Relay
`plugins.toml`, then restart the host:
@@ -386,6 +389,10 @@ Within the routed request's trace:
for the selected and served model. The judge response contains the returned
verdict; the decision mark records what Switchyard did with it.
+For System One or OpenAI Decisions, inspect the input and output on
+`libsy.decision_client_call` directly. The captured request uses the provider's
+final `state` or `input` field and encoded questions.
+
For an [ATOF export](https://docs.nvidia.com/nemo/relay/configure-plugins/observability/atof),
the request appears directly in the attempt start event's `data`, and the
response in the matching end event's `data`. Their `uuid` values match;
@@ -393,19 +400,19 @@ response in the matching end event's `data`. Their `uuid` values match;
captured requests with:
```bash
-jq 'select(.name == "libsy.upstream_attempt" and .scope_category == "start" and .data.body != null) | .data.body | fromjson' events.jsonl
+jq 'select((.name == "libsy.upstream_attempt" or .name == "libsy.decision_client_call") and .scope_category == "start" and .data.body != null) | .data.body | fromjson' events.jsonl
```
Find omitted request or response bodies with:
```bash
-jq 'select(.name == "libsy.upstream_attempt" and .data.truncated == true) | {uuid, scope_category}' events.jsonl
+jq 'select((.name == "libsy.upstream_attempt" or .name == "libsy.decision_client_call") and .data.truncated == true) | {uuid, scope_category}' events.jsonl
```
Capture follows routing-phase HTTP calls, which include LLM classifier/judge
-calls and may include other model calls used while deciding a route. It excludes
-completion-phase payloads, streamed response bodies, HTTP error bodies, and
-non-LLM decision-client payloads such as SystemOne. This option belongs to the
+calls, typed decision-client calls, and other model calls used while deciding a
+route. It excludes completion-phase payloads, streamed response bodies, and
+HTTP error bodies. This option belongs to the
native Relay plugin; enabling standalone Switchyard TRACE logging does not
enable payload capture.
@@ -441,6 +448,7 @@ flowchart LR
llm["LLM call"]
call["libsy.client_call<br/>routing or completion"]
attempt["libsy.upstream_attempt<br/>one per HTTP attempt"]
+ decision_call["libsy.decision_client_call<br/>typed decision"]
requested["mark:switchyard.routing.requested<br/>zero duration"]
calls["mark:switchyard.routing.llm_call<br/>zero duration"]
overhead["mark:switchyard.routing.overhead<br/>zero duration"]
@@ -449,6 +457,7 @@ flowchart LR
agent --> llm
llm --> call
call --> attempt
+ llm --> decision_call
agent --> requested
agent --> calls
agent --> overhead
@@ -539,7 +548,8 @@ Configure delivery through Relay's
By default, Switchyard routing telemetry excludes request and response content.
Enabling [routing payload capture](#inspect-classifier-and-judge-payloads) adds
internal routing requests and successful buffered responses to
-`libsy.upstream_attempt` scopes. Routing marks remain payload-free.
+`libsy.upstream_attempt` and `libsy.decision_client_call` scopes. Routing marks
+remain payload-free.
Relay's LLM events can capture request and response data according to Relay's
[input and output event semantics](https://docs.nvidia.com/nemo/relay/about-nemo-relay/concepts/events#input-and-output-payloads),
independently of these Switchyard marks.