Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::pin::Pin;
use std::sync::{Arc, Mutex};
use futures_util::{Stream, StreamExt};
use http::StatusCode;
use nemo_relay_plugin::{
DataSchema, Json, LlmRequest as RelayRequest, LogSeverity, MetricKind, MetricMeasurement,
MetricValueType, PluginRuntime,
};
use serde_json::{Map, json};
use switchyard_llm_client::{ModelCallObservation, RunObservation, RunObserver};
use switchyard_protocol::{
LlmClientError, LlmResponse, LlmResponseChunk, LlmStreamError, Metadata, ProviderExtensions,
Request, Response, Usage, WireFormat,
};
use switchyard_runner::{
ProviderKeyRedactor, Route, RouteErrorSummary, Runner, stream_error_summary,
};
use switchyard_translation::{TranslationEngine, encode_stream_with_extensions};
use crate::config::SwitchyardConfig;
use crate::span_capture::CapturedSpan;
use crate::translation;
#[cfg(test)]
#[path = "redaction_tests.rs"]
mod redaction_tests;
const ROUTING_MARK_SCHEMA_VERSION: &str = "1";
const MAX_EVIDENCE_STRING_BYTES: usize = 64;
#[derive(Debug)]
pub(crate) struct RoutingMark {
pub(crate) name: String,
pub(crate) data: Json,
pub(crate) metadata: Json,
pub(crate) severity: Option<LogSeverity>,
}
impl RoutingMark {
fn data_schema(&self) -> DataSchema {
DataSchema {
name: self.name.clone(),
version: ROUTING_MARK_SCHEMA_VERSION.into(),
}
}
}
#[derive(Debug)]
pub(crate) struct RoutingMetric {
pub(crate) name: String,
pub(crate) measurements: Vec<MetricMeasurement>,
pub(crate) metadata: Json,
}
#[derive(Debug)]
pub(crate) enum RoutingEvent {
Mark(RoutingMark),
Metric(RoutingMetric),
Span(CapturedSpan),
}
struct MetricDescriptor<'a> {
name: &'a str,
kind: MetricKind,
value_type: MetricValueType,
unit: Option<&'a str>,
description: &'a str,
}
pub(crate) type ReturnedEventStream = Pin<Box<dyn Stream<Item = Result<Json, String>> + Send>>;
pub(crate) type RoutingEventEmitter = Arc<dyn Fn(RoutingEvent) + Send + Sync>;
pub(crate) struct Execution<T> {
pub(crate) result: Result<T, String>,
pub(crate) events: Vec<RoutingEvent>,
}
struct RoutedResponse {
response: Response,
outcome_fields: Map<String, Json>,
}
pub(crate) struct SwitchyardRuntime {
runner: Runner,
translation: TranslationEngine,
redactor: Arc<ProviderKeyRedactor>,
}
impl SwitchyardRuntime {
pub(crate) fn new(config: SwitchyardConfig) -> Result<Self, String> {
let runner = config.load_runner()?;
let redactor = Arc::new(ProviderKeyRedactor::new(runner.provider_api_keys()));
Ok(Self {
runner,
translation: TranslationEngine::default(),
redactor,
})
}
pub(crate) fn redactor(&self) -> &ProviderKeyRedactor {
&self.redactor
}
pub(crate) fn manages_model(&self, model: &str) -> bool {
self.runner.route(model).is_some()
}
pub(crate) fn decode_request(
&self,
inbound: WireFormat,
request: RelayRequest,
streaming: bool,
) -> Result<Request, String> {
let mut llm_request = translation::decode_request(&self.translation, inbound, &request)
.map_err(|error| self.redactor.text(error))?;
llm_request.stream = streaming;
let headers = string_headers(&request.headers);
switchyard_translation::normalize_compaction(&mut llm_request, Some(&headers));
let mut metadata = Metadata::from_headers(&headers);
let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id")
&& headers
.get("x-nemo-relay-source")
.and_then(|value| value.to_str().ok())
== Some("gateway")
&& metadata.session_id.as_deref() == Some("gateway-gateway");
if relay_gateway_placeholder {
metadata.session_id = None;
}
metadata.http_headers = Some(headers);
Ok(Request {
llm_request,
raw_request: Some(request.content),
metadata: Some(metadata),
})
}
pub(crate) async fn execute_buffered(
&self,
inbound: WireFormat,
request: Request,
) -> Execution<Json> {
let request_extensions = request.llm_request.extensions.clone();
let Execution { result, mut events } = self.execute(inbound, request).await;
let (result, finalization_failed) = match result {
Ok(routed) => {
let result = finalize_buffered_response(
&self.translation,
inbound,
routed.response,
&request_extensions,
);
let failed = result.is_err();
(result, failed)
}
Err(error) => (Err(error), false),
};
if finalization_failed {
self.error_mark(&mut events, "response_finalization", None);
}
self.sanitize_execution(Execution {
result: result.map(|value| self.redactor.value(value)),
events,
})
}
pub(crate) async fn execute_stream(
&self,
inbound: WireFormat,
request: Request,
emit_event: RoutingEventEmitter,
) -> Execution<ReturnedEventStream> {
let emit_event = self.sanitized_emitter(emit_event);
let request_extensions = request.llm_request.extensions.clone();
let Execution { result, mut events } = self.execute(inbound, request).await;
let (result, finalization_failed) = match result {
Ok(routed) => {
let metadata = event_metadata(&events).unwrap_or_else(|| Json::Object(Map::new()));
let result = returned_events(
routed.response,
inbound,
&request_extensions,
metadata,
routed.outcome_fields,
emit_event,
);
let failed = result.is_err();
(result, failed)
}
Err(error) => (Err(error), false),
};
if finalization_failed {
self.error_mark(&mut events, "response_finalization", None);
}
let redactor = Arc::clone(&self.redactor);
self.sanitize_execution(Execution {
result: result.map(|stream| {
Box::pin(stream.map(move |item| {
item.map(|value| redactor.value(value))
.map_err(|error| redactor.text(error))
})) as ReturnedEventStream
}),
events,
})
}
fn sanitize_execution<T>(&self, execution: Execution<T>) -> Execution<T> {
Execution {
result: execution.result.map_err(|error| self.redactor.text(error)),
events: execution
.events
.into_iter()
.map(|event| sanitize_event(&self.redactor, event))
.collect(),
}
}
fn sanitized_emitter(&self, emit_event: RoutingEventEmitter) -> RoutingEventEmitter {
let redactor = Arc::clone(&self.redactor);
Arc::new(move |event| emit_event(sanitize_event(&redactor, event)))
}
async fn execute(&self, inbound: WireFormat, request: Request) -> Execution<RoutedResponse> {
let Some(route) = self.route(&request) else {
return Execution {
result: Err("Switchyard has no route for this request model".into()),
events: Vec::new(),
};
};
let metadata = identity_metadata(request.metadata.as_ref());
let mut events = vec![RoutingEvent::Mark(RoutingMark {
name: "switchyard.routing.requested".into(),
data: json!({"algorithm": route.algorithm_name()}),
metadata: metadata.clone(),
severity: Some(LogSeverity::Info),
})];
events.push(request_metric(route.algorithm_name(), metadata.clone()));
if let Err(error) = route.check_caller_format(inbound) {
self.error_mark(&mut events, "caller_format", None);
return Execution {
result: Err(format!("Switchyard caller format is incompatible: {error}")),
events,
};
}
let observations = Arc::new(Mutex::new(Vec::new()));
let observed = Arc::clone(&observations);
let observer: RunObserver = Arc::new(move |observation| {
observed
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.push(observation);
});
match route.execute(request, Some(observer)).await {
Ok(mut output) => {
let outcome_fields = self.emit_observations(
&mut events,
take_observations(&observations),
&metadata,
route,
);
let served_target = output.response.served_model().cloned();
let served_model = served_target
.as_ref()
.map(|target| route.provider_model(target));
let mut decision_outcome_fields = outcome_fields.clone();
let mut data = json!({
"algorithm": route.algorithm_name(),
"selected_model": route.provider_model(&output.selected_model).as_str(),
"served_model": served_model,
"selected_target": output.selected_model.as_str(),
"served_target": served_target,
"reasoning_effort": served_target.as_ref().and_then(|target| route.target(target)).and_then(|target| target.reasoning_effort.as_deref()),
"fallback_used": served_target.as_ref().map(|target| target != &output.selected_model),
});
if let Json::Object(data) = &mut data {
data.append(&mut decision_outcome_fields);
}
events.push(RoutingEvent::Mark(RoutingMark {
name: "switchyard.routing.decision".into(),
data,
metadata,
severity: Some(LogSeverity::Info),
}));
if let Some(model) = served_model {
output.response.set_served_model(model);
}
Execution {
result: Ok(RoutedResponse {
response: output.response,
outcome_fields,
}),
events,
}
}
Err(error) => {
let outcome_fields = self.emit_observations(
&mut events,
take_observations(&observations),
&metadata,
route,
);
self.route_execution_error_mark(
&mut events,
&error.execution_error_summary(),
None,
outcome_fields,
);
Execution {
result: Err("Switchyard route execution failed".into()),
events,
}
}
}
}
fn route(&self, request: &Request) -> Option<&Route> {
request
.llm_request
.model
.as_deref()
.and_then(|model| self.runner.route(model))
}
fn emit_observations(
&self,
events: &mut Vec<RoutingEvent>,
observations: Vec<RunObservation>,
metadata: &Json,
route: &Route,
) -> Map<String, Json> {
let mut call_index = 0;
let mut outcome_fields = Map::new();
for mut observation in observations {
// Relay's model fields and token metrics retain provider identities.
match &mut observation {
RunObservation::LlmCall(call)
| RunObservation::DecisionCall(call)
| RunObservation::AnswerCall(call) => {
call.selected_model = route.provider_model(&call.selected_model).clone();
}
_ => {}
}
match observation {
RunObservation::Outcome(outcome) => {
outcome_fields.insert(
"outcome_id".into(),
Json::String(outcome.outcome_id().into()),
);
if let Some(evidence) = evidence_for_mark(outcome.evidence) {
outcome_fields.insert("evidence".into(), evidence);
}
}
RunObservation::LlmCall(call) => {
call_index += 1;
self.routing_call_events(
events,
call,
call_index,
metadata,
"switchyard.routing.llm_call",
);
}
RunObservation::DecisionCall(call) => {
call_index += 1;
self.routing_call_events(
events,
call,
call_index,
metadata,
"switchyard.routing.decision_call",
);
}
RunObservation::RoutingOverhead(duration) => {
let latency_ms = duration.as_secs_f64() * 1_000.0;
events.push(RoutingEvent::Mark(RoutingMark {
name: "switchyard.routing.overhead".into(),
data: json!({"latency_ms": latency_ms}),
metadata: metadata.clone(),
severity: Some(LogSeverity::Info),
}));
events.push(routing_overhead_metric(latency_ms, metadata.clone()));
}
RunObservation::AnswerCall(call) => {
call_index += 1;
let outcome = if call.is_success { "ok" } else { "error" };
let latency_ms = call.duration.as_secs_f64() * 1_000.0;
events.push(RoutingEvent::Mark(RoutingMark {
name: "switchyard.routing.llm_call".into(),
data: json!({
"call_index": call_index,
"selected_model": call.selected_model.as_str(),
"call_role": "answer",
"outcome": outcome,
"latency_ms": latency_ms,
}),
metadata: metadata.clone(),
severity: Some(LogSeverity::Debug),
}));
events.extend(token_usage_metrics("answer", &call, metadata));
}
}
}
outcome_fields
}
fn routing_call_events(
&self,
events: &mut Vec<RoutingEvent>,
call: ModelCallObservation,
call_index: usize,
metadata: &Json,
mark_name: &str,
) {
let outcome = if call.is_success { "ok" } else { "error" };
let latency_ms = call.duration.as_secs_f64() * 1_000.0;
let token_metrics = token_usage_metrics("routing", &call, metadata);
events.push(RoutingEvent::Mark(RoutingMark {
name: mark_name.into(),
data: json!({
"call_index": call_index,
"selected_model": call.selected_model.as_str(),
"call_role": "routing",
"outcome": outcome,
"latency_ms": latency_ms,
}),
metadata: metadata.clone(),
severity: Some(LogSeverity::Debug),
}));
events.extend(routing_call_metrics(outcome, latency_ms, metadata.clone()));
events.extend(token_metrics);
}
fn error_mark(
&self,
events: &mut Vec<RoutingEvent>,
failure_kind: &str,
metadata: Option<&Json>,
) {
let metadata = metadata
.cloned()
.unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));
events.push(RoutingEvent::Mark(RoutingMark {
name: "switchyard.routing.error".into(),
data: json!({"failure_kind": failure_kind}),
metadata: metadata.clone(),
severity: Some(LogSeverity::Error),
}));
events.push(failure_metric(failure_kind, None, None, None, metadata));
}
fn route_execution_error_mark(
&self,
events: &mut Vec<RoutingEvent>,
summary: &RouteErrorSummary,
metadata: Option<&Json>,
outcome_fields: Map<String, Json>,
) {
let metadata = metadata
.cloned()
.unwrap_or_else(|| event_metadata(events).unwrap_or_else(|| Json::Object(Map::new())));
events.extend(route_execution_error_events(
summary,
metadata,
outcome_fields,
));
}
}
/// Projects documented, bounded evidence into Relay mark data.
fn evidence_for_mark(evidence: Option<Json>) -> Option<Json> {
let Some(Json::Object(mut evidence)) = evidence else {
return None;
};
evidence.retain(|name, value| match name.as_str() {
"source" | "verdict" | "trigger" | "reason_code" => value
.as_str()
.is_some_and(|value| value.len() <= MAX_EVIDENCE_STRING_BYTES),
"score" | "confidence" | "threshold" | "capable_score" | "capable_threshold" => {
value.is_number()
}
_ => false,
});
(!evidence.is_empty()).then_some(Json::Object(evidence))
}
pub(crate) fn emit_events(emit_event: &RoutingEventEmitter, events: Vec<RoutingEvent>) {
for event in events {
emit_event(event);
}
}
pub(crate) fn emit_event(
runtime: &PluginRuntime,
redactor: &ProviderKeyRedactor,
event: RoutingEvent,
) {
let result = match event {
RoutingEvent::Mark(mark) => {
let data_schema = mark.data_schema();
runtime
.emit_mark_with_options(
&mark.name,
Some(&mark.data),
Some(&mark.metadata),
Some(&data_schema),
mark.severity,
)
.map_err(|error| ("routing mark", mark.name, error))
}
RoutingEvent::Metric(metric) => runtime
.emit_metric(&metric.name, metric.measurements, Some(&metric.metadata))
.map_err(|error| ("routing metric", metric.name, error)),
RoutingEvent::Span(span) => span
.emit(runtime)
.map_err(|error| ("routing span", "libsy".into(), error)),
};
if let Err((kind, name, error)) = result {
let message = redactor.text(format!(
"Switchyard could not emit {kind} {name:?}: {error}"
));
eprintln!("{message}");
}
}
// Sanitize both synchronous marks and metrics emitted while a stream is polled.
fn sanitize_event(redactor: &ProviderKeyRedactor, mut event: RoutingEvent) -> RoutingEvent {
match &mut event {
RoutingEvent::Mark(mark) => {
mark.data = redactor.value(std::mem::take(&mut mark.data));
mark.metadata = redactor.value(std::mem::take(&mut mark.metadata));
}
RoutingEvent::Metric(metric) => {
metric.metadata = redactor.value(std::mem::take(&mut metric.metadata));
for measurement in &mut metric.measurements {
measurement.attributes = measurement
.attributes
.take()
.map(|value| redactor.value(value));
}
}
RoutingEvent::Span(span) => span.sanitize(redactor),
}
event
}
fn take_observations(observations: &Mutex<Vec<RunObservation>>) -> Vec<RunObservation> {
std::mem::take(
&mut *observations
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()),
)
}
fn finalize_buffered_response(
translation_engine: &TranslationEngine,
inbound: WireFormat,
response: Response,
request_extensions: &ProviderExtensions,
) -> Result<Json, String> {
let LlmResponse::Agg(response) = response.llm_response else {
return Err("Switchyard returned a stream for a buffered request".into());
};
translation::encode_response(translation_engine, inbound, &response, request_extensions)
}
fn returned_events(
response: Response,
inbound: WireFormat,
request_extensions: &ProviderExtensions,
metadata: Json,
mut outcome_fields: Map<String, Json>,
emit_event: RoutingEventEmitter,
) -> Result<ReturnedEventStream, String> {
let served_model = response.served_model().cloned();
let (chunks, observe_stream_usage) = match response.llm_response {
LlmResponse::Agg(response) => (response.into_stream(), false),
LlmResponse::Stream(chunks) => (chunks, true),
};
let mut latest_usage = None;
let mut terminal_seen = false;
let mut usage_emitted = false;
let mut stream_failed = false;
let mut failure_emitted = false;
let chunks = Box::pin(chunks.map(move |item| {
let in_band_error = item
.as_ref()
.ok()
.and_then(|event| normalized_stream_error(event.normalized()));
let stream_error = in_band_error.as_ref().or_else(|| item.as_ref().err());
stream_failed |= stream_error.is_some();
if let Some(error) = stream_error
&& !failure_emitted
{
for event in route_execution_error_events(
&stream_error_summary(error, served_model.as_ref()),
metadata.clone(),
std::mem::take(&mut outcome_fields),
) {
emit_event(event);
}
failure_emitted = true;
}
if observe_stream_usage && !stream_failed && !usage_emitted {
if let Ok(event) = &item {
for chunk in event.normalized() {
match chunk {
LlmResponseChunk::Usage(usage) => latest_usage = Some(usage.clone()),
LlmResponseChunk::MessageStop { .. } => terminal_seen = true,
_ => {}
}
}
}
if terminal_seen
&& let (Some(model), Some(usage)) = (served_model.as_ref(), latest_usage.as_ref())
{
for event in usage_metrics("answer", model.as_str(), usage, &metadata) {
emit_event(event);
}
usage_emitted = true;
}
}
item
}));
let events = encode_stream_with_extensions(chunks, inbound, None, request_extensions)
.map_err(|error| format!("Switchyard response stream setup failed: {error}"))?;
Ok(Box::pin(
events.map(|item| item.map_err(relay_stream_error)),
))
}
/// Produces a telemetry-only error summary source for a normalized in-band failure.
///
/// The translation layer retains the original provider error event for client delivery;
/// this conversion is only used to classify the failure without putting its message in marks.
fn normalized_stream_error(chunks: &[LlmResponseChunk]) -> Option<LlmClientError> {
chunks.iter().find_map(|chunk| match chunk {
LlmResponseChunk::DecodeError { message } => {
Some(LlmClientError::ResponseTranslation(message.clone()))
}
LlmResponseChunk::StreamError { message } => Some(LlmClientError::UpstreamHttp {
status: StatusCode::BAD_GATEWAY,
body: message.clone(),
}),
_ => None,
})
}
/// Converts translation errors to Relay's string-only stream-error boundary.
///
/// `LlmStreamError::Upstream` retains a provider-shaped JSON value so its caller can
/// make protocol decisions, but neither that value nor a client error's source message
/// is safe for Relay's outer observability scope. Preserve the error result while
/// exposing only a stable failure class.
fn relay_stream_error(error: LlmStreamError) -> String {
match error {
LlmStreamError::Upstream(_) => "Switchyard upstream stream error".into(),
LlmStreamError::Client(error) => format!(
"Switchyard stream error: {}",
stream_error_summary(&error, None).kind.as_str()
),
}
}
fn route_execution_error_mark(
summary: &RouteErrorSummary,
metadata: Json,
mut outcome_fields: Map<String, Json>,
) -> RoutingMark {
let mut data = json!({
"failure_kind": "route_execution",
"category": summary.kind.as_str(),
"phase": summary.phase.as_str(),
"upstream_status": summary.upstream_status,
"target": summary.target.as_ref().map(|target| target.as_str()),
});
if let Json::Object(data) = &mut data {
data.append(&mut outcome_fields);
}
RoutingMark {
name: "switchyard.routing.error".into(),
data,
metadata,
severity: Some(LogSeverity::Error),
}
}
fn route_execution_error_events(
summary: &RouteErrorSummary,
metadata: Json,
outcome_fields: Map<String, Json>,
) -> Vec<RoutingEvent> {
vec![
RoutingEvent::Mark(route_execution_error_mark(
summary,
metadata.clone(),
outcome_fields,
)),
failure_metric(
"route_execution",
Some(summary.kind.as_str()),
Some(summary.phase.as_str()),
summary.upstream_status,
metadata,
),
]
}
fn event_metadata(events: &[RoutingEvent]) -> Option<Json> {
events.iter().find_map(|event| match event {
RoutingEvent::Mark(mark) => Some(mark.metadata.clone()),
_ => None,
})
}
fn request_metric(algorithm: &str, metadata: Json) -> RoutingEvent {
counter_metric(
"switchyard.routing.requests",
"Requests managed by Switchyard routing.",
json!({"algorithm": algorithm}),
metadata,
)
}
fn routing_call_metrics(outcome: &str, latency_ms: f64, metadata: Json) -> [RoutingEvent; 2] {
let attributes = json!({"outcome": outcome});
[
counter_metric(
"switchyard.routing.llm_calls",
"Switchyard model calls made while routing.",
attributes.clone(),
metadata.clone(),
),
histogram_metric(
"switchyard.routing.llm_call.duration",
"Duration of Switchyard model calls made while routing.",
latency_ms,
attributes,
metadata,
),
]
}
fn routing_overhead_metric(latency_ms: f64, metadata: Json) -> RoutingEvent {
histogram_metric(
"switchyard.routing.overhead",
"Time needed to produce the Switchyard routing outcome, including routing model calls.",
latency_ms,
json!({}),
metadata,
)
}
fn token_usage_metrics(
call_role: &str,
call: &ModelCallObservation,
metadata: &Json,
) -> Vec<RoutingEvent> {
let Some(usage) = call.usage.as_ref() else {
return Vec::new();
};
usage_metrics(call_role, call.selected_model.as_str(), usage, metadata)
}
fn usage_metrics(
call_role: &str,
target_model: &str,
usage: &Usage,
metadata: &Json,
) -> Vec<RoutingEvent> {
[
("input", usage.input_tokens),
("cached_input", usage.cached_input_tokens()),
("cache_creation_input", usage.cache_creation_input_tokens()),
("output", usage.output_tokens),
("reasoning", usage.reasoning_tokens),
("total", usage.total_tokens),
]
.into_iter()
.filter_map(|(token_type, value)| {
value.map(|value| {
metric(
MetricDescriptor {
name: "switchyard.routing.llm_tokens",
kind: MetricKind::Counter,
value_type: MetricValueType::U64,
unit: Some("{token}"),
description: "Normalized tokens used by Switchyard model calls.",
},
json!(value),
json!({
"call_role": call_role,
"target_model": target_model,
"token_type": token_type,
}),
metadata.clone(),
)
})
})
.collect()
}
fn failure_metric(
failure_kind: &str,
category: Option<&str>,
phase: Option<&str>,
upstream_status: Option<u16>,
metadata: Json,
) -> RoutingEvent {
let mut attributes = Map::new();
attributes.insert("failure_kind".into(), Json::String(failure_kind.into()));
if let Some(category) = category {
attributes.insert("category".into(), Json::String(category.into()));
}
if let Some(phase) = phase {
attributes.insert("phase".into(), Json::String(phase.into()));
}
if let Some(upstream_status) = upstream_status {
attributes.insert("upstream_status".into(), Json::from(upstream_status));
}
counter_metric(
"switchyard.routing.failures",
"Terminal Switchyard routing failures.",
Json::Object(attributes),
metadata,
)
}
fn counter_metric(name: &str, description: &str, attributes: Json, metadata: Json) -> RoutingEvent {
metric(
MetricDescriptor {
name,
kind: MetricKind::Counter,
value_type: MetricValueType::U64,
unit: Some("{event}"),
description,
},
json!(1),
attributes,
metadata,
)
}
fn histogram_metric(
name: &str,
description: &str,
value: f64,
attributes: Json,
metadata: Json,
) -> RoutingEvent {
metric(
MetricDescriptor {
name,
kind: MetricKind::Histogram,
value_type: MetricValueType::F64,
unit: Some("ms"),
description,
},
json!(value),
attributes,
metadata,
)
}
fn metric(
descriptor: MetricDescriptor<'_>,
value: Json,
attributes: Json,
metadata: Json,
) -> RoutingEvent {
RoutingEvent::Metric(RoutingMetric {
name: descriptor.name.into(),
measurements: vec![MetricMeasurement {
name: descriptor.name.into(),
kind: descriptor.kind,
value_type: descriptor.value_type,
value,
unit: descriptor.unit.map(Into::into),
description: Some(descriptor.description.into()),
attributes: Some(attributes),
boundaries: None,
}],
metadata,
})
}
fn string_headers(headers: &Map<String, Json>) -> http::HeaderMap {
let mut parsed = http::HeaderMap::with_capacity(headers.len());
for (name, value) in headers {
let Some(value) = value.as_str() else {
continue;
};
let (Ok(name), Ok(value)) = (
http::HeaderName::from_bytes(name.as_bytes()),
http::HeaderValue::from_str(value),
) else {
continue;
};
parsed.insert(name, value);
}
parsed
}
fn identity_metadata(metadata: Option<&Metadata>) -> Json {
json!({
"session_id": metadata.and_then(|value| value.session_id.as_deref()),
"agent_id": metadata.and_then(|value| value.agent_id.as_deref()),
"parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()),
"task_id": metadata.and_then(|value| value.task_id.as_deref()),
"turn_id": metadata.and_then(|value| value.turn_id.as_deref()),
"correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()),
})
}
#[cfg(test)]
mod tests {
use std::collections::{BTreeMap, HashMap};
use switchyard_runner::RuntimeModels;
use switchyard_llm_client::ClientRouter;
use switchyard_protocol::{LlmClientError, LlmResponseStreamEvent, ModelId, Usage};
use switchyard_runner::{AlgorithmSpec, ModelCapabilities, RunnerError};
use wiremock::matchers::{body_partial_json, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
use super::*;
fn runtime_for(model: &str) -> SwitchyardRuntime {
let algorithm = AlgorithmSpec::Noop {}
.build("relay", &BTreeMap::new())
.expect("noop route should build");
let route = Route::new(
algorithm,
ClientRouter::new(HashMap::new()),
None,
ModelCapabilities::default(),
None,
None,
Vec::new(),
RuntimeModels::default(),
);
SwitchyardRuntime {
runner: Runner::new(vec![(ModelId::from(model), route)]),
translation: TranslationEngine::default(),
redactor: Arc::default(),
}
}
fn runtime_for_target(format: &str, base_url: &str) -> SwitchyardRuntime {
let deployment = json!({
"schema_version": 1,
"llm_clients": {
"target": {
"format": format,
"base_url": base_url,
}
},
"targets": {
"default": {
"id": "target/model",
"llm_client": "target",
}
},
"routes": {
"default": {
"id": "switchyard/default",
"type": "passthrough",
"target": "default",
}
}
});
SwitchyardRuntime::new(crate::config::SwitchyardConfig {
capture_routing_payloads: false,
max_routing_payload_bytes: crate::config::DEFAULT_ROUTING_PAYLOAD_BYTES,
priority: 0,
switchyard_config_path: None,
switchyard_config: Some(deployment.as_object().unwrap().clone()),
})
.expect("cross-format runtime should load")
}
fn namespaced_responses_request() -> RelayRequest {
RelayRequest {
headers: Map::new(),
content: json!({
"model": "switchyard/default",
"input": "Search the docs",
"tools": [{
"type": "namespace",
"name": "mcp__docs",
"tools": [{
"type": "function",
"name": "search",
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
},
}],
}],
}),
}
}
#[test]
fn only_configured_route_models_are_managed() {
let runtime = runtime_for("switchyard");
assert!(runtime.manages_model("switchyard"));
assert!(!runtime.manages_model("other"));
}
#[tokio::test]
async fn responses_requests_use_the_configured_target_format() {
for (format, endpoint, upstream_response) in [
(
"openai_chat",
"/v1/chat/completions",
json!({
"id": "chatcmpl_test",
"object": "chat.completion",
"model": "target/model",
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": "ok"},
"finish_reason": "stop",
}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}),
),
(
"anthropic_messages",
"/v1/messages",
json!({
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "target/model",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
}),
),
] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path(endpoint))
.and(body_partial_json(json!({"model": "target/model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(upstream_response))
.expect(1)
.mount(&server)
.await;
let runtime = runtime_for_target(format, &format!("{}/v1", server.uri()));
let request = RelayRequest {
headers: Map::new(),
content: json!({
"model": "switchyard/default",
"input": "hello",
}),
};
let decoded = runtime
.decode_request(WireFormat::OpenAiResponses, request, false)
.expect("Responses request should decode");
assert_eq!(
decoded
.metadata
.as_ref()
.and_then(|metadata| metadata.wire_format),
None
);
let execution = runtime
.execute_buffered(WireFormat::OpenAiResponses, decoded)
.await;
let decision = execution
.events
.iter()
.find_map(|event| match event {
RoutingEvent::Mark(mark) if mark.name == "switchyard.routing.decision" => {
Some(&mark.data)
}
_ => None,
})
.expect("decision mark should be emitted");
assert_eq!(decision["selected_model"], "target/model");
assert_eq!(decision["selected_target"], "default");
assert_eq!(decision["served_model"], "target/model");
assert_eq!(decision["served_target"], "default");
assert_eq!(decision["fallback_used"], false);
assert!(decision["outcome_id"].is_string());
let response = execution.result.expect("target call should succeed");
assert_eq!(response["object"], "response");
assert_eq!(response["model"], "target/model");
}
}
// Falling back between efforts must remain visible even when provider IDs match.
#[tokio::test]
async fn same_model_fallback_keeps_provider_fields_and_target_details() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(body_partial_json(json!({"reasoning_effort": "medium"})))
.respond_with(ResponseTemplate::new(503))
.expect(1)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(body_partial_json(json!({"model": "shared/model", "reasoning_effort": "max"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "reply", "model": "shared/model",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 2, "completion_tokens": 1}
}))).expect(1).mount(&server).await;
let runner = Runner::from_toml(&format!(
r#"
schema_version = 1
[llm_clients.upstream]
format = "openai_chat"
base_url = "{}/v1"
max_retries = 0
[targets.efficient]
id = "shared/model"
llm_client = "upstream"
reasoning_effort = "medium"
[targets.capable]
id = "shared/model"
llm_client = "upstream"
reasoning_effort = "max"
[routes.test]
id = "route"
type = "random"
targets = ["efficient", "capable"]
weights = [1000000, 1]
seed = 17
"#,
server.uri()
))
.unwrap();
let runtime = SwitchyardRuntime {
runner,
translation: TranslationEngine::default(),
redactor: Arc::default(),
};
let request = runtime.decode_request(WireFormat::OpenAiChat, RelayRequest {
headers: Map::new(), content: json!({"model": "route", "messages": [{"role": "user", "content": "hello"}]}),
}, false).unwrap();
let execution = runtime
.execute_buffered(WireFormat::OpenAiChat, request)
.await;
assert_eq!(execution.result.unwrap()["model"], "shared/model");
let mut saw_decision = false;
for event in execution.events {
if let RoutingEvent::Mark(mark) = event {
if mark.name == "switchyard.routing.decision" {
saw_decision = true;
assert_eq!(mark.data["selected_model"], "shared/model");
assert_eq!(mark.data["served_model"], "shared/model");
assert_eq!(mark.data["selected_target"], "efficient");
assert_eq!(mark.data["served_target"], "capable");
assert_eq!(mark.data["reasoning_effort"], "max");
assert_eq!(mark.data["fallback_used"], true);
} else if mark.name == "switchyard.routing.llm_call" {
assert_eq!(mark.data["selected_model"], "shared/model");
}
}
}
assert!(saw_decision);
}
// Routing succeeds before answer candidates exhaust, so the error keeps its metadata.
#[tokio::test]
async fn failed_answer_keeps_routing_outcome_in_error_mark() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.respond_with(ResponseTemplate::new(503).set_body_string("unavailable"))
.expect(2)
.mount(&server)
.await;
let deployment = json!({
"schema_version": 1,
"llm_clients": {
"local": {
"format": "openai_chat",
"base_url": format!("{}/v1", server.uri()),
"max_retries": 0,
}
},
"targets": {
"strong": {"id": "poc/strong", "llm_client": "local"},
"weak": {"id": "poc/weak", "llm_client": "local"},
},
"routes": {
"stage": {
"id": "switchyard/stage",
"type": "stage_router",
"capable_target": "strong",
"efficient_target": "weak",
"picker": "efficient_first",
"confidence_threshold": 0.5,
}
},
});
let runtime = SwitchyardRuntime::new(crate::config::SwitchyardConfig {
capture_routing_payloads: false,
max_routing_payload_bytes: crate::config::DEFAULT_ROUTING_PAYLOAD_BYTES,
priority: 0,
switchyard_config_path: None,
switchyard_config: Some(
deployment
.as_object()
.expect("deployment should be an object")
.clone(),
),
})
.expect("stage runtime should load");
let request = runtime
.decode_request(
WireFormat::OpenAiChat,
RelayRequest {
headers: Map::new(),
content: json!({
"model": "switchyard/stage",
"messages": [{"role": "user", "content": "hello"}],
}),
},
false,
)
.expect("request should decode");
let execution = runtime
.execute_buffered(WireFormat::OpenAiChat, request)
.await;
assert!(execution.result.is_err());
let error = execution
.events
.iter()
.find_map(|event| match event {
RoutingEvent::Mark(mark) if mark.name == "switchyard.routing.error" => {
Some(&mark.data)
}
_ => None,
})
.expect("error mark should be emitted");
assert!(error["outcome_id"].is_string());
assert_eq!(error["evidence"], json!({"source": "fall_open"}));
}
#[tokio::test]
async fn buffered_responses_restore_codex_tool_namespaces() {
let server = MockServer::start().await;
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(json!({"model": "target/model"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"id": "chatcmpl_tool",
"object": "chat.completion",
"model": "target/model",
"choices": [{
"index": 0,
"message": {
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_search",
"type": "function",
"function": {
"name": "mcp__docs__search",
"arguments": "{\"query\":\"routing\"}",
},
}],
},
"finish_reason": "tool_calls",
}],
"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5},
})))
.expect(1)
.mount(&server)
.await;
let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri()));
let request = runtime
.decode_request(
WireFormat::OpenAiResponses,
namespaced_responses_request(),
false,
)
.expect("namespaced Responses request should decode");
let response = runtime
.execute_buffered(WireFormat::OpenAiResponses, request)
.await
.result
.expect("buffered target call should succeed");
let tool_call = response["output"]
.as_array()
.and_then(|output| output.iter().find(|item| item["type"] == "function_call"))
.expect("Responses output should contain a function call");
assert_eq!(tool_call["name"], "search");
assert_eq!(tool_call["namespace"], "mcp__docs");
}
#[tokio::test]
async fn streaming_responses_restore_codex_tool_namespaces() {
let server = MockServer::start().await;
let sse = concat!(
"data: {\"id\":\"chatcmpl_tool\",\"object\":\"chat.completion.chunk\",\"model\":\"target/model\",\"choices\":[{\"index\":0,\"delta\":{\"role\":\"assistant\",\"tool_calls\":[{\"index\":0,\"id\":\"call_search\",\"type\":\"function\",\"function\":{\"name\":\"mcp__docs__search\",\"arguments\":\"{\\\"query\\\":\\\"routing\\\"}\"}}]},\"finish_reason\":null}]}\n\n",
"data: {\"id\":\"chatcmpl_tool\",\"object\":\"chat.completion.chunk\",\"model\":\"target/model\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"tool_calls\"}]}\n\n",
"data: [DONE]\n\n",
);
Mock::given(method("POST"))
.and(path("/v1/chat/completions"))
.and(body_partial_json(
json!({"model": "target/model", "stream": true}),
))
.respond_with(ResponseTemplate::new(200).set_body_raw(sse, "text/event-stream"))
.expect(1)
.mount(&server)
.await;
let runtime = runtime_for_target("openai_chat", &format!("{}/v1", server.uri()));
let request = runtime
.decode_request(
WireFormat::OpenAiResponses,
namespaced_responses_request(),
true,
)
.expect("namespaced streaming Responses request should decode");
let stream = runtime
.execute_stream(WireFormat::OpenAiResponses, request, Arc::new(|_| {}))
.await
.result
.expect("streaming target call should succeed");
let events = stream
.collect::<Vec<_>>()
.await
.into_iter()
.collect::<Result<Vec<_>, _>>()
.expect("response stream should encode");
for event_type in ["response.output_item.added", "response.output_item.done"] {
let item = events
.iter()
.find(|event| event["type"] == event_type)
.map(|event| &event["item"])
.unwrap_or_else(|| panic!("stream produced no {event_type}"));
assert_eq!(item["name"], "search", "{event_type}");
assert_eq!(item["namespace"], "mcp__docs", "{event_type}");
}
let completed = events
.iter()
.find(|event| event["type"] == "response.completed")
.expect("stream produced no response.completed event");
assert_eq!(completed["response"]["output"][0]["name"], "search");
assert_eq!(completed["response"]["output"][0]["namespace"], "mcp__docs");
}
#[test]
fn execution_failure_mark_uses_the_safe_runner_summary() {
let secret = "provider response body";
let error = RunnerError::Client(LlmClientError::ContextWindowExceeded {
model: ModelId::from("weak"),
message: secret.into(),
});
let mark = route_execution_error_mark(
&error.execution_error_summary(),
json!({"session_id": "session"}),
Map::new(),
);
assert_eq!(mark.name, "switchyard.routing.error");
assert_eq!(mark.data["failure_kind"], "route_execution");
assert_eq!(mark.data["category"], "context_window_exceeded");
assert_eq!(mark.data["phase"], "before_response");
assert_eq!(mark.data["target"], "weak");
assert_eq!(mark.data["upstream_status"], Json::Null);
assert_eq!(mark.severity, Some(LogSeverity::Error));
assert_eq!(mark.data_schema().name, "switchyard.routing.error");
assert_eq!(mark.data_schema().version, "1");
assert!(!mark.data.to_string().contains(secret));
}
// Custom algorithms can supply evidence, so the Relay boundary filters it.
#[test]
fn decision_evidence_keeps_only_documented_bounded_fields() {
let evidence = evidence_for_mark(Some(json!({
"source": "llm-classifier",
"score": 0.9,
"confidence": "wrong type",
"threshold": 0.5,
"capable_score": 0.7,
"capable_threshold": 0.6,
"verdict": "continue",
"trigger": "turn",
"reason_code": "x".repeat(MAX_EVIDENCE_STRING_BYTES + 1),
"unknown": "ignored",
"prompt": {"text": "ignored"},
})));
assert_eq!(
evidence,
Some(json!({
"source": "llm-classifier",
"score": 0.9,
"threshold": 0.5,
"capable_score": 0.7,
"capable_threshold": 0.6,
"verdict": "continue",
"trigger": "turn",
}))
);
}
#[test]
fn routing_observations_emit_debug_marks_and_metrics() {
let runtime = runtime_for("switchyard");
let mut events = Vec::new();
runtime.emit_observations(
&mut events,
vec![
RunObservation::LlmCall(ModelCallObservation {
selected_model: ModelId::from("routing-model"),
is_success: false,
duration: std::time::Duration::from_millis(12),
usage: Some(Usage {
input_tokens: Some(4),
..Usage::default()
}),
}),
RunObservation::RoutingOverhead(std::time::Duration::from_millis(3)),
],
&json!({"session_id": "session"}),
runtime.runner.route("switchyard").unwrap(),
);
assert_eq!(events.len(), 6);
let RoutingEvent::Mark(call_mark) = &events[0] else {
panic!("first event should be the routing call mark");
};
assert_eq!(call_mark.name, "switchyard.routing.llm_call");
assert_eq!(call_mark.severity, Some(LogSeverity::Debug));
assert_eq!(call_mark.data["outcome"], "error");
assert!(call_mark.data.get("usage").is_none());
let RoutingEvent::Metric(call_count) = &events[1] else {
panic!("second event should be the routing call counter");
};
assert_eq!(call_count.name, "switchyard.routing.llm_calls");
assert_eq!(call_count.measurements[0].kind, MetricKind::Counter);
assert_eq!(
call_count.measurements[0].attributes,
Some(json!({"outcome": "error"}))
);
let RoutingEvent::Metric(call_duration) = &events[2] else {
panic!("third event should be the routing call histogram");
};
assert_eq!(call_duration.name, "switchyard.routing.llm_call.duration");
assert_eq!(call_duration.measurements[0].kind, MetricKind::Histogram);
assert_eq!(call_duration.measurements[0].value, json!(12.0));
let RoutingEvent::Metric(tokens) = &events[3] else {
panic!("fourth event should be the routing token counter");
};
assert_eq!(tokens.name, "switchyard.routing.llm_tokens");
let RoutingEvent::Mark(overhead_mark) = &events[4] else {
panic!("fifth event should be the routing overhead mark");
};
assert_eq!(overhead_mark.severity, Some(LogSeverity::Info));
let RoutingEvent::Metric(overhead) = &events[5] else {
panic!("sixth event should be the routing overhead histogram");
};
assert_eq!(overhead.name, "switchyard.routing.overhead");
assert_eq!(overhead.measurements[0].attributes, Some(json!({})));
}
#[test]
fn request_and_failure_metrics_use_bounded_attributes() {
let RoutingEvent::Metric(request) = request_metric("stage_router", json!({})) else {
panic!("request should be a metric");
};
assert_eq!(request.name, "switchyard.routing.requests");
assert_eq!(
request.measurements[0].attributes,
Some(json!({"algorithm": "stage_router"}))
);
let RoutingEvent::Metric(failure) = failure_metric(
"route_execution",
Some("upstream_http"),
Some("before_response"),
Some(503),
json!({}),
) else {
panic!("failure should be a metric");
};
assert_eq!(failure.name, "switchyard.routing.failures");
assert_eq!(
failure.measurements[0].attributes,
Some(json!({
"failure_kind": "route_execution",
"category": "upstream_http",
"phase": "before_response",
"upstream_status": 503,
}))
);
}
#[test]
fn token_usage_metrics_distinguish_routing_and_answer_targets() {
let call = ModelCallObservation {
selected_model: ModelId::from("judge-model"),
is_success: true,
duration: std::time::Duration::from_millis(1),
usage: Some(Usage {
input_tokens: Some(11),
cache: Usage::cache_details(Some(3), Some(2)),
output_tokens: Some(7),
total_tokens: Some(23),
reasoning_tokens: Some(5),
provider_cost: None,
}),
};
let routing = token_usage_metrics("routing", &call, &json!({"session_id": "session"}));
assert_eq!(routing.len(), 6);
for event in &routing {
let RoutingEvent::Metric(metric) = event else {
panic!("token usage should be emitted as a metric");
};
assert_eq!(metric.name, "switchyard.routing.llm_tokens");
assert_eq!(metric.measurements[0].kind, MetricKind::Counter);
assert_eq!(metric.measurements[0].unit.as_deref(), Some("{token}"));
assert_eq!(
metric.measurements[0].attributes.as_ref().unwrap()["call_role"],
"routing"
);
assert_eq!(
metric.measurements[0].attributes.as_ref().unwrap()["target_model"],
"judge-model"
);
}
let token_values = routing
.iter()
.map(|event| {
let RoutingEvent::Metric(metric) = event else {
panic!("token usage should be emitted as a metric");
};
metric.measurements[0].value.clone()
})
.collect::<Vec<_>>();
assert_eq!(
token_values,
vec![json!(11), json!(3), json!(2), json!(7), json!(5), json!(23)]
);
let answer = token_usage_metrics("answer", &call, &json!({}));
let RoutingEvent::Metric(metric) = &answer[0] else {
panic!("answer usage should be emitted as a metric");
};
assert_eq!(
metric.measurements[0].attributes.as_ref().unwrap()["call_role"],
"answer"
);
}
#[test]
fn answer_observations_emit_call_marks_and_token_metrics() {
let runtime = runtime_for("switchyard");
let mut events = Vec::new();
runtime.emit_observations(
&mut events,
vec![
RunObservation::AnswerCall(ModelCallObservation {
selected_model: ModelId::from("weak-target"),
is_success: false,
duration: std::time::Duration::from_millis(2),
usage: None,
}),
RunObservation::AnswerCall(ModelCallObservation {
selected_model: ModelId::from("strong-target"),
is_success: true,
duration: std::time::Duration::from_millis(3),
usage: Some(Usage {
output_tokens: Some(9),
..Usage::default()
}),
}),
],
&json!({}),
runtime.runner.route("switchyard").unwrap(),
);
assert_eq!(events.len(), 3);
for (event, index, target, outcome, latency_ms) in [
(&events[0], 1, "weak-target", "error", 2.0),
(&events[1], 2, "strong-target", "ok", 3.0),
] {
let RoutingEvent::Mark(mark) = event else {
panic!("answer observation should emit a call mark");
};
assert_eq!(mark.name, "switchyard.routing.llm_call");
assert_eq!(mark.data["call_index"], index);
assert_eq!(mark.data["selected_model"], target);
assert_eq!(mark.data["call_role"], "answer");
assert_eq!(mark.data["outcome"], outcome);
assert_eq!(mark.data["latency_ms"], latency_ms);
assert_eq!(mark.severity, Some(LogSeverity::Debug));
}
let RoutingEvent::Metric(metric) = &events[2] else {
panic!("successful answer usage should emit a token metric");
};
assert_eq!(metric.name, "switchyard.routing.llm_tokens");
assert_eq!(
metric.measurements[0].attributes,
Some(json!({
"call_role": "answer",
"target_model": "strong-target",
"token_type": "output",
}))
);
}
#[tokio::test]
async fn stream_usage_emits_answer_token_metrics() {
let response = Response {
llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([
Ok(LlmResponseStreamEvent::new(vec![
LlmResponseChunk::MessageStop { reason: None },
])),
Ok(LlmResponseStreamEvent::new(vec![LlmResponseChunk::Usage(
Usage {
input_tokens: Some(10),
output_tokens: Some(3),
total_tokens: Some(13),
..Usage::default()
},
)])),
]))),
metadata: Some(Metadata {
served_model: Some(ModelId::from("selected-target")),
..Default::default()
}),
upstream_headers: http::HeaderMap::new(),
};
let captured = Arc::new(Mutex::new(Vec::new()));
let emitted = Arc::clone(&captured);
let mut stream = returned_events(
response,
WireFormat::OpenAiChat,
&ProviderExtensions::default(),
json!({"session_id": "session"}),
Map::new(),
Arc::new(move |event| emitted.lock().unwrap().push(event)),
)
.expect("stream setup should succeed");
assert!(captured.lock().unwrap().is_empty());
assert!(stream.next().await.expect("encoded stream event").is_ok());
assert!(captured.lock().unwrap().is_empty());
assert!(stream.next().await.expect("encoded usage event").is_ok());
let events = captured.lock().unwrap();
assert_eq!(events.len(), 3);
let token_types = events
.iter()
.map(|event| {
let RoutingEvent::Metric(metric) = event else {
panic!("stream usage should emit only metrics");
};
assert_eq!(metric.name, "switchyard.routing.llm_tokens");
assert_eq!(metric.metadata, json!({"session_id": "session"}));
let attributes = metric.measurements[0].attributes.as_ref().unwrap();
assert_eq!(attributes["call_role"], "answer");
assert_eq!(attributes["target_model"], "selected-target");
attributes["token_type"].as_str().unwrap().to_string()
})
.collect::<Vec<_>>();
assert_eq!(token_types, ["input", "output", "total"]);
}
#[tokio::test]
async fn aggregate_response_stream_does_not_reemit_usage_metrics() {
let mut aggregate =
switchyard_protocol::text_response(Some("selected-target".into()), "ok");
aggregate.usage = Usage {
input_tokens: Some(10),
output_tokens: Some(3),
total_tokens: Some(13),
..Usage::default()
};
let response = Response {
llm_response: LlmResponse::Agg(aggregate),
metadata: Some(Metadata {
served_model: Some(ModelId::from("selected-target")),
..Default::default()
}),
upstream_headers: http::HeaderMap::new(),
};
let captured = Arc::new(Mutex::new(Vec::new()));
let emitted = Arc::clone(&captured);
let stream = returned_events(
response,
WireFormat::OpenAiChat,
&ProviderExtensions::default(),
json!({}),
Map::new(),
Arc::new(move |event| emitted.lock().unwrap().push(event)),
)
.expect("stream setup should succeed");
let events = stream.collect::<Vec<_>>().await;
assert!(events.iter().all(Result::is_ok));
assert!(captured.lock().unwrap().is_empty());
}
#[tokio::test]
async fn stream_failure_emits_a_safe_failure_mark() {
let secret = "provider response body";
let response = Response {
llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Err::<
LlmResponseStreamEvent,
LlmClientError,
>(
LlmClientError::ContextWindowExceeded {
model: ModelId::from("weak"),
message: secret.into(),
},
)]))),
metadata: Some(Metadata {
served_model: Some(ModelId::from("strong")),
..Default::default()
}),
upstream_headers: http::HeaderMap::new(),
};
let captured = Arc::new(Mutex::new(Vec::new()));
let emitted = Arc::clone(&captured);
// Routing has completed before this response stream reports its failure.
let stream = returned_events(
response,
WireFormat::OpenAiChat,
&ProviderExtensions::default(),
json!({"session_id": "session"}),
Map::from_iter([
("outcome_id".into(), json!("outcome-test")),
("evidence".into(), json!({"source": "fall_open"})),
]),
Arc::new(move |mark| emitted.lock().unwrap().push(mark)),
)
.expect("stream setup should succeed");
let events = stream.collect::<Vec<_>>().await;
assert!(events[0].is_err());
assert!(
!events[0]
.as_ref()
.expect_err("stream should fail")
.contains(secret)
);
let events = captured.lock().unwrap();
assert_eq!(events.len(), 2);
let RoutingEvent::Mark(mark) = &events[0] else {
panic!("first event should be the safe failure mark");
};
assert_eq!(mark.data["category"], "context_window_exceeded");
assert_eq!(mark.data["phase"], "during_stream");
assert_eq!(mark.data["target"], "strong");
assert_eq!(mark.data["outcome_id"], "outcome-test");
assert_eq!(mark.data["evidence"], json!({"source": "fall_open"}));
assert_eq!(mark.severity, Some(LogSeverity::Error));
assert!(!mark.data.to_string().contains(secret));
let RoutingEvent::Metric(metric) = &events[1] else {
panic!("second event should be the failure counter");
};
assert_eq!(metric.name, "switchyard.routing.failures");
assert_eq!(
metric.measurements[0].attributes,
Some(json!({
"failure_kind": "route_execution",
"category": "context_window_exceeded",
"phase": "during_stream",
}))
);
}
async fn assert_normalized_stream_failure(
chunk: LlmResponseChunk,
expected_category: &str,
expected_status: Option<u16>,
) {
let secret = "provider response body";
let response = Response {
llm_response: LlmResponse::Stream(Box::pin(futures_util::stream::iter([Ok(
LlmResponseStreamEvent::new(vec![chunk]),
)]))),
metadata: Some(Metadata {
served_model: Some(ModelId::from("selected-target")),
..Default::default()
}),
upstream_headers: http::HeaderMap::new(),
};
let captured = Arc::new(Mutex::new(Vec::new()));
let emitted = Arc::clone(&captured);
let stream = returned_events(
response,
WireFormat::OpenAiChat,
&ProviderExtensions::default(),
json!({"session_id": "session"}),
Map::new(),
Arc::new(move |event| emitted.lock().unwrap().push(event)),
)
.expect("stream setup should succeed");
let client_events = stream.collect::<Vec<_>>().await;
assert_eq!(client_events.len(), 1);
assert!(client_events[0].is_err());
assert!(
!client_events[0]
.as_ref()
.expect_err("stream should fail")
.contains(secret)
);
let events = captured.lock().unwrap();
assert_eq!(events.len(), 2, "one mark and one counter should emit");
let RoutingEvent::Mark(mark) = &events[0] else {
panic!("first event should be the safe failure mark");
};
assert_eq!(mark.data["category"], expected_category);
assert_eq!(mark.data["phase"], "during_stream");
assert_eq!(
mark.data["upstream_status"],
expected_status.map_or(Json::Null, Json::from)
);
assert_eq!(mark.data["target"], "selected-target");
assert!(!mark.data.to_string().contains(secret));
let RoutingEvent::Metric(metric) = &events[1] else {
panic!("second event should be the failure counter");
};
assert_eq!(metric.name, "switchyard.routing.failures");
let mut expected_attributes = json!({
"failure_kind": "route_execution",
"category": expected_category,
"phase": "during_stream",
});
if let Some(status) = expected_status {
expected_attributes
.as_object_mut()
.expect("expected metric attributes should be an object")
.insert("upstream_status".into(), json!(status));
}
assert_eq!(metric.measurements[0].attributes, Some(expected_attributes));
}
#[tokio::test]
async fn normalized_stream_error_emits_failure_telemetry_once() {
assert_normalized_stream_failure(
LlmResponseChunk::StreamError {
message: "provider response body".into(),
},
"upstream_http",
Some(502),
)
.await;
}
#[tokio::test]
async fn normalized_decode_error_emits_failure_telemetry_once() {
assert_normalized_stream_failure(
LlmResponseChunk::DecodeError {
message: "provider response body".into(),
},
"response_translation",
None,
)
.await;
}
}