Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(relay): preserve per-call usage and session identity (#1062)

nachiketb-nvidiaf5b973c8b2bd
diff --git a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
index 10fc1c473..94037b024 100644
--- a/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
+++ b/crates/switchyard-nemo-relay-plugin/src/span_capture.rs
@@ -126,25 +126,95 @@ impl CapturedSpan {
}
pub(crate) fn emit(self, runtime: &PluginRuntime) -> Result<(), String> {
- let metadata = Json::Object(self.fields);
- let status = span_status(&metadata);
+ let mut metadata = Json::Object(self.fields);
+ let output = if self.name == CLIENT_CALL {
+ response_data(&metadata)
+ } else {
+ None
+ };
+ if let Some(session_id) = metadata.get("gen_ai.conversation.id").cloned() {
+ metadata["session_id"] = session_id;
+ }
+ if let Some(status) = span_status(&metadata) {
+ metadata["otel.status_code"] = status["otel.status_code"].clone();
+ }
+ let input = metadata
+ .get("gen_ai.request.model")
+ .map(|model| serde_json::json!({"model": model}));
let mut scope = runtime.scope_at(
self.name,
relay_scope_type(self.name),
None,
Some(&metadata),
- None,
+ input.as_ref(),
self.started_at,
)?;
let mut child_result = Ok(());
for child in self.children {
child_result = child_result.and(child.emit(runtime));
}
- let close_result = scope.close_at(None, status.as_ref(), self.ended_at);
+ let close_result = scope.close_at(output.as_ref(), Some(&metadata), self.ended_at);
child_result.and(close_result)
}
}
+fn response_data(fields: &Json) -> Option<Json> {
+ let mut output = Map::new();
+ for (source, target) in [
+ ("gen_ai.response.id", "id"),
+ ("gen_ai.response.model", "model"),
+ ] {
+ if let Some(value) = fields.get(source) {
+ output.insert(target.into(), value.clone());
+ }
+ }
+ let mut usage = Map::new();
+ for (source, target) in [
+ ("gen_ai.usage.output_tokens", "output_tokens"),
+ (
+ "gen_ai.usage.cache_read.input_tokens",
+ "cache_read_input_tokens",
+ ),
+ (
+ "gen_ai.usage.cache_creation.input_tokens",
+ "cache_creation_input_tokens",
+ ),
+ ] {
+ if let Some(value) = fields.get(source) {
+ usage.insert(target.into(), value.clone());
+ }
+ }
+ if let Some(input) = fields
+ .get("gen_ai.usage.input_tokens")
+ .and_then(Json::as_u64)
+ {
+ let cached = usage
+ .get("cache_read_input_tokens")
+ .and_then(Json::as_u64)
+ .unwrap_or(0);
+ let created = usage
+ .get("cache_creation_input_tokens")
+ .and_then(Json::as_u64)
+ .unwrap_or(0);
+ usage.insert(
+ "input_tokens".into(),
+ input.saturating_sub(cached).saturating_sub(created).into(),
+ );
+ }
+ if !usage.is_empty() {
+ output.insert("usage".into(), Json::Object(usage));
+ }
+ if output.is_empty() {
+ return None;
+ }
+ // Relay decodes scope output with provider codecs. A content-free Messages
+ // response preserves separate cache-read and cache-creation token counts.
+ output.insert("type".into(), Json::String("message".into()));
+ output.insert("role".into(), Json::String("assistant".into()));
+ output.insert("content".into(), Json::Array(Vec::new()));
+ Some(Json::Object(output))
+}
+
/// Captures Switchyard client spans for one plugin registration.
#[derive(Clone)]
pub(crate) struct SpanCapture {
@@ -256,6 +326,22 @@ fn is_safe_field(name: &str) -> bool {
| "switchyard.candidate_count"
| "switchyard.call_phase"
| "selected_model"
+ | "gen_ai.request.model"
+ | "gen_ai.request.stream"
+ | "gen_ai.request.temperature"
+ | "gen_ai.request.top_p"
+ | "gen_ai.request.top_k"
+ | "gen_ai.request.max_tokens"
+ | "gen_ai.request.reasoning.level"
+ | "gen_ai.output.type"
+ | "gen_ai.conversation.id"
+ | "gen_ai.response.id"
+ | "gen_ai.response.model"
+ | "gen_ai.usage.input_tokens"
+ | "gen_ai.usage.output_tokens"
+ | "gen_ai.usage.cache_read.input_tokens"
+ | "gen_ai.usage.cache_creation.input_tokens"
+ | "gen_ai.usage.reasoning.output_tokens"
| "model"
| "wire_format"
| "attempt"