Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Integration tests for the crate's observability layer: OpenTelemetry metrics
//! recorded through the global meter provider, and `tracing` spans/logs captured
//! by a global subscriber.
//!
//! Both telemetry sinks are process-global, so they are installed exactly once
//! for this test binary and every assertion filters by a test-unique algorithm
//! and model name. Counters are cumulative across flushes; the helpers take the
//! latest (max) matching data point.
use std::collections::BTreeMap;
use std::fmt;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::Duration;
use async_trait::async_trait;
use futures::StreamExt;
use opentelemetry::trace::TracerProvider as _;
use opentelemetry::{Array as OtelArray, Value as OtelValue};
use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ResourceMetrics};
use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
use parking_lot::Mutex;
use serde_json::json;
use tracing::field::{Field, Visit};
use tracing::span::{Attributes, Id, Record};
use tracing::{Event, Subscriber};
use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::Layer;
use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt};
use tracing_subscriber::registry::LookupSpan;
use switchyard_libsy::{
Algorithm, CapabilityJudgeConfig, ClassifierContractConfig, ClassifyTrigger,
DeescalationConfig, Driver, EscalationJudgeConfig, LibsyError, LlmCapabilityConfig,
LlmClassifierConfig, LlmTaskClassifier, PickerMode, RoutingOutcome, RuntimeModels, StageRouter,
StageRouterConfig, Step, TaskClassifierConfig,
};
use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver};
use switchyard_protocol::{Category, ModelId};
use switchyard_protocol::{
ContentBlock, LlmRequest, LlmResponse, Message, Metadata, Request, Response, Role,
RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat,
};
use switchyard_protocol::{
LlmClientError, LlmResponseChunk, LlmResponseStreamEvent, StopReason, text_request,
text_response,
};
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct TestError(&'static str);
fn test_error(message: &'static str) -> LibsyError {
LibsyError::external("test", TestError(message))
}
/// One captured span: its name, contextual parent span name, and fields
/// (creation-time fields merged with later `Span::record` updates).
#[derive(Clone, Debug, Default)]
struct SpanRecord {
name: String,
parent: Option<String>,
fields: BTreeMap<String, String>,
}
/// One captured event (log line): its target and fields, `message` included.
#[derive(Clone, Debug, Default)]
struct EventRecord {
target: String,
level: String,
fields: BTreeMap<String, String>,
}
/// Shared store the capture layer writes into and tests read from.
#[derive(Clone, Default)]
struct CaptureStore {
spans: Arc<Mutex<BTreeMap<u64, SpanRecord>>>,
events: Arc<Mutex<Vec<EventRecord>>>,
}
impl CaptureStore {
fn spans(&self) -> Vec<SpanRecord> {
self.spans.lock().values().cloned().collect()
}
fn events(&self) -> Vec<EventRecord> {
self.events.lock().clone()
}
}
/// Renders every field type into a string map so assertions can use `contains`.
struct FieldVisitor<'a>(&'a mut BTreeMap<String, String>);
impl Visit for FieldVisitor<'_> {
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
self.0
.insert(field.name().to_string(), format!("{value:?}"));
}
fn record_str(&mut self, field: &Field, value: &str) {
self.0.insert(field.name().to_string(), value.to_string());
}
fn record_u64(&mut self, field: &Field, value: u64) {
self.0.insert(field.name().to_string(), value.to_string());
}
fn record_i64(&mut self, field: &Field, value: i64) {
self.0.insert(field.name().to_string(), value.to_string());
}
fn record_f64(&mut self, field: &Field, value: f64) {
self.0.insert(field.name().to_string(), value.to_string());
}
}
/// Subscriber layer capturing spans (with contextual parents and recorded
/// fields) and events into a [`CaptureStore`].
struct CaptureLayer {
store: CaptureStore,
}
impl<S> Layer<S> for CaptureLayer
where
S: Subscriber + for<'a> LookupSpan<'a>,
{
fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: LayerContext<'_, S>) {
let mut fields = BTreeMap::new();
attrs.record(&mut FieldVisitor(&mut fields));
// Resolve the parent the same way tracing does: an explicit parent wins,
// otherwise the contextually current span (if any) at creation time.
let parent = if let Some(parent_id) = attrs.parent() {
ctx.span(parent_id).map(|span| span.name().to_string())
} else if attrs.is_contextual() {
ctx.lookup_current().map(|span| span.name().to_string())
} else {
None
};
self.store.spans.lock().insert(
id.into_u64(),
SpanRecord {
name: attrs.metadata().name().to_string(),
parent,
fields,
},
);
}
fn on_record(&self, id: &Id, values: &Record<'_>, _ctx: LayerContext<'_, S>) {
if let Some(record) = self.store.spans.lock().get_mut(&id.into_u64()) {
values.record(&mut FieldVisitor(&mut record.fields));
}
}
fn on_event(&self, event: &Event<'_>, _ctx: LayerContext<'_, S>) {
let mut fields = BTreeMap::new();
event.record(&mut FieldVisitor(&mut fields));
self.store.events.lock().push(EventRecord {
target: event.metadata().target().to_string(),
level: event.metadata().level().to_string(),
fields,
});
}
}
/// Installs the process-global telemetry sinks once: an in-memory OTel metric
/// pipeline behind the global meter provider, and the capture layer as the
/// global tracing subscriber.
type Telemetry = (
CaptureStore,
InMemoryMetricExporter,
SdkMeterProvider,
InMemorySpanExporter,
SdkTracerProvider,
);
fn telemetry() -> &'static Telemetry {
static TELEMETRY: OnceLock<Telemetry> = OnceLock::new();
TELEMETRY.get_or_init(|| {
let exporter = InMemoryMetricExporter::default();
let reader = PeriodicReader::builder(exporter.clone()).build();
let provider = SdkMeterProvider::builder().with_reader(reader).build();
opentelemetry::global::set_meter_provider(provider.clone());
switchyard_llm_client::initialize_metrics();
let span_exporter = InMemorySpanExporter::default();
let tracer_provider = SdkTracerProvider::builder()
.with_simple_exporter(span_exporter.clone())
.build();
let tracer = tracer_provider.tracer("switchyard-observability-test");
let store = CaptureStore::default();
let otel_layer: OpenTelemetryLayer<_, _> =
tracing_opentelemetry::layer().with_tracer(tracer);
let subscriber = tracing_subscriber::registry()
.with(CaptureLayer {
store: store.clone(),
})
.with(otel_layer);
if tracing::subscriber::set_global_default(subscriber).is_err() {
panic!("a global tracing subscriber was already installed in this test binary");
}
(store, exporter, provider, span_exporter, tracer_provider)
})
}
/// The tests in this file must not overlap because metrics are global.
/// There is no Rust/cargo-native way of saying this (people use `serial_test` crate) so use a
/// lock.
/// Each file in `tests/` (integration tests) runs as a separate test process, so we are not
/// concerned with interactions with tests in other files.
fn serialize_test() -> &'static tokio::sync::Mutex<()> {
static LOCK: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| tokio::sync::Mutex::new(()))
}
/// Flushes the metric pipeline and returns every exported snapshot.
fn flushed_metrics(
exporter: &InMemoryMetricExporter,
provider: &SdkMeterProvider,
) -> Vec<ResourceMetrics> {
if let Err(error) = provider.force_flush() {
panic!("force_flush failed: {error}");
}
match exporter.get_finished_metrics() {
Ok(metrics) => metrics,
Err(error) => panic!("get_finished_metrics failed: {error}"),
}
}
/// True when the data point carries every wanted `key=value` attribute.
fn attributes_match<'a>(
mut attributes: impl Iterator<Item = &'a opentelemetry::KeyValue>,
wanted: &[(&str, &str)],
) -> bool {
let present: Vec<(String, String)> = attributes
.by_ref()
.map(|kv| (kv.key.as_str().to_string(), kv.value.as_str().to_string()))
.collect();
wanted
.iter()
.all(|(key, value)| present.iter().any(|(k, v)| k == key && v == value))
}
/// Latest (max) value for a `switchyard`-scoped metric across snapshots, with
/// `extract` pulling the matching data points' values out of one metric's
/// aggregated data. Counters and histogram counts are cumulative, so the max
/// across snapshots is the most recent value.
fn latest_metric_value(
snapshots: &[ResourceMetrics],
name: &str,
extract: impl Fn(&AggregatedMetrics) -> Vec<u64>,
) -> Option<u64> {
snapshots
.iter()
.flat_map(|snapshot| snapshot.scope_metrics())
.filter(|scope| scope.scope().name() == "switchyard")
.flat_map(|scope| scope.metrics())
.filter(|metric| metric.name() == name)
.flat_map(|metric| extract(metric.data()))
.max()
}
/// Latest cumulative value of a `u64` counter for the given attribute set.
fn u64_counter_value(
snapshots: &[ResourceMetrics],
name: &str,
wanted: &[(&str, &str)],
) -> Option<u64> {
latest_metric_value(snapshots, name, |data| match data {
AggregatedMetrics::U64(MetricData::Sum(sum)) => sum
.data_points()
.filter(|point| attributes_match(point.attributes(), wanted))
.map(|point| point.value())
.collect(),
_ => Vec::new(),
})
}
/// Latest cumulative sample count of an `f64` histogram for the attribute set.
fn f64_histogram_count(
snapshots: &[ResourceMetrics],
name: &str,
wanted: &[(&str, &str)],
) -> Option<u64> {
latest_metric_value(snapshots, name, |data| match data {
AggregatedMetrics::F64(MetricData::Histogram(histogram)) => histogram
.data_points()
.filter(|point| attributes_match(point.attributes(), wanted))
.map(|point| point.count())
.collect(),
_ => Vec::new(),
})
}
/// Latest cumulative sample sum of an `f64` histogram, in whole milliseconds.
fn f64_histogram_sum_ms(
snapshots: &[ResourceMetrics],
name: &str,
wanted: &[(&str, &str)],
) -> Option<u64> {
latest_metric_value(snapshots, name, |data| match data {
AggregatedMetrics::F64(MetricData::Histogram(histogram)) => histogram
.data_points()
.filter(|point| attributes_match(point.attributes(), wanted))
.map(|point| point.sum() as u64)
.collect(),
_ => Vec::new(),
})
}
/// Most recent value of an `i64` up-down counter for the given attribute set.
///
/// Unlike the counter and histogram helpers, this takes the value from the last
/// snapshot that carries the metric rather than the max across snapshots: an
/// up-down counter falls as well as rises, so a max would report a past peak and
/// never observe the return to zero.
fn i64_up_down_counter_value(
snapshots: &[ResourceMetrics],
name: &str,
wanted: &[(&str, &str)],
) -> Option<i64> {
snapshots
.iter()
.flat_map(|snapshot| snapshot.scope_metrics())
.filter(|scope| scope.scope().name() == "switchyard")
.flat_map(|scope| scope.metrics())
.filter(|metric| metric.name() == name)
.filter_map(|metric| match metric.data() {
AggregatedMetrics::I64(MetricData::Sum(sum)) => sum
.data_points()
.filter(|point| attributes_match(point.attributes(), wanted))
.map(|point| point.value())
.last(),
_ => None,
})
.last()
}
/// Latest value of a `u64` observable gauge.
fn u64_gauge_value(snapshots: &[ResourceMetrics], name: &str) -> Option<u64> {
latest_metric_value(snapshots, name, |data| match data {
AggregatedMetrics::U64(MetricData::Gauge(gauge)) => {
gauge.data_points().map(|point| point.value()).collect()
}
_ => Vec::new(),
})
}
/// Client that answers every call with a fixed token [`Usage`].
struct UsageClient {
usage: Usage,
}
/// Client that returns a weak classifier verdict. The delays let a test tell
/// classifier time apart from routed-call time.
struct ClassifierClient {
classifier_model_id: ModelId,
classifier_delay: Duration,
routed_delay: Duration,
}
/// Fails the first efficient answer so candidate fallback can be compared with affinity.
struct AffinityFallbackClient {
calls: Mutex<Vec<ModelId>>,
efficient_available: AtomicBool,
}
#[async_trait]
impl RoutedLlmClient for AffinityFallbackClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
let model = request.model_id().unwrap_or_default();
self.calls.lock().push(model.clone());
if model == "affinity-fallback-judge" {
return Ok(Response {
llm_response: LlmResponse::Agg(text_response(
Some(model.to_string()),
r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#,
)),
metadata: None,
upstream_headers: http::HeaderMap::new(),
});
}
if model == "affinity-fallback-weak" && !self.efficient_available.load(Ordering::Relaxed) {
return Err(LlmClientError::ContextWindowExceeded {
model,
message: "too long".to_string(),
});
}
Ok(Response {
llm_response: LlmResponse::Agg(text_response(Some(model.to_string()), "answer")),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
}
}
#[async_trait]
impl RoutedLlmClient for ClassifierClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
let model_id = request.model_id().unwrap_or_default();
let completion = if model_id != self.classifier_model_id {
tokio::time::sleep(self.routed_delay).await;
"routed response"
} else {
tokio::time::sleep(self.classifier_delay).await;
r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#
};
Ok(Response {
llm_response: LlmResponse::Agg(text_response(Some(model_id.to_string()), completion)),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
}
}
// Conversation text that a provider may quote in an upstream error body.
const LEAKED_CONTENT: &str = "patient name is Jane Doe";
enum JudgeOutcome {
CallFailure,
Reply(&'static str),
StreamDecodeFailure,
}
/// Returns one configured judge outcome and serves the selected target normally.
struct JudgeClient {
judge_model: ModelId,
outcome: JudgeOutcome,
}
#[async_trait]
impl RoutedLlmClient for JudgeClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
if request.model_id().as_deref().unwrap_or_default() != self.judge_model {
return Ok(Response {
llm_response: LlmResponse::Agg(text_response(
request.model_id().map(|id| id.to_string()),
"routed response",
)),
metadata: None,
upstream_headers: http::HeaderMap::new(),
});
}
match &self.outcome {
JudgeOutcome::CallFailure => Err(LlmClientError::UpstreamHttp {
status: http::StatusCode::INTERNAL_SERVER_ERROR,
body: format!(r#"{{"error":{{"message":"server error: {LEAKED_CONTENT}"}}}}"#),
}),
JudgeOutcome::Reply(text) => Ok(Response {
llm_response: LlmResponse::Agg(text_response(None, *text)),
metadata: None,
upstream_headers: http::HeaderMap::new(),
}),
JudgeOutcome::StreamDecodeFailure => Ok(Response {
llm_response: LlmResponse::Stream(
futures::stream::iter([Ok(LlmResponseStreamEvent::new(vec![
LlmResponseChunk::DecodeError {
message: "bad judge chunk".to_string(),
},
]))])
.boxed(),
),
metadata: None,
upstream_headers: http::HeaderMap::new(),
}),
}
}
}
#[async_trait]
impl RoutedLlmClient for UsageClient {
async fn call(
&self,
request: Request,
) -> Result<Response, switchyard_protocol::LlmClientError> {
let mut response = text_response(
request.model_id().map(|s| s.to_string()),
"observed response",
);
response.id = Some("obs-response-1".to_string());
response.usage = self.usage.clone();
response.outputs[0].stop_reason = Some(StopReason::EndTurn);
Ok(Response {
llm_response: LlmResponse::Agg(response),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
}
}
/// Selects the first target and leaves the terminal call to the client.
struct SingleCallAlgo {
name: String,
target_set: Vec<String>,
metadata: Option<switchyard_libsy::OutcomeMetadata>,
}
#[async_trait]
impl Algorithm for SingleCallAlgo {
fn name(&self) -> &str {
&self.name
}
async fn route(
self: Arc<Self>,
_driver: Driver,
request: Request,
) -> switchyard_libsy::Result<RoutingOutcome> {
let target = self
.target_set
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
tracing::info!("picked '{target}'");
// Include configured fallbacks so the exported model order can be checked.
let mut outcome = RoutingOutcome::route_to(
target.into(),
self.target_set
.iter()
.skip(1)
.cloned()
.map(ModelId::from)
.collect(),
request,
);
outcome.metadata = self.metadata.clone();
Ok(outcome)
}
}
/// Makes one routing-time model call and returns its response as the answer.
struct RoutingCallAlgo {
name: String,
target: ModelId,
}
#[async_trait]
impl Algorithm for RoutingCallAlgo {
fn name(&self) -> &str {
&self.name
}
async fn route(
self: Arc<Self>,
driver: Driver,
request: Request,
) -> switchyard_libsy::Result<RoutingOutcome> {
let response = driver
.call_model(request.clone(), vec![self.target.clone()])
.await?;
Ok(RoutingOutcome::answered(
self.target.clone(),
request,
response,
))
}
}
fn request_with_metadata(session_id: &str, correlation_id: &str) -> Request {
Request {
llm_request: text_request(Some("auto".to_string()), "hi"),
raw_request: None,
metadata: Some(Metadata {
session_id: Some(session_id.to_string()),
correlation_id: Some(correlation_id.to_string()),
task_kind: Some("code_review".to_string()),
agent_role: Some("reviewer".to_string()),
extra_metadata: Some(BTreeMap::from([(
"tenant".to_string(),
"obs-tenant-1".to_string(),
)])),
..Metadata::default()
}),
}
}
fn algo(name: &str, model: &str) -> Arc<dyn Algorithm> {
Arc::new(SingleCallAlgo {
name: name.to_string(),
target_set: vec![model.to_string()],
metadata: None,
})
}
/// Drives `algorithm` to completion with `client` serving every target, the way a
/// single-provider host does.
async fn run(
algorithm: Arc<dyn Algorithm>,
client: Arc<dyn RoutedLlmClient>,
request: Request,
) -> switchyard_libsy::Result<(ModelId, Response)> {
switchyard_llm_client::run(
algorithm,
ClientRouter::single(client),
request,
Arc::new(RuntimeModels::default()),
None,
)
.await
}
fn classifier_router() -> switchyard_libsy::Result<Arc<dyn Algorithm>> {
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Capability {
config: TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold: 0.5,
..LlmCapabilityConfig::default()
}),
fail_open: false,
..TaskClassifierConfig::default()
},
},
)?))
}
fn classifier_models(
judge_model: &str,
efficient_model: &str,
capable_model: &str,
) -> Arc<RuntimeModels> {
Arc::new(RuntimeModels::new(
[
(Category::Judge, vec![judge_model.into()]),
(Category::Efficient, vec![efficient_model.into()]),
(Category::Capable, vec![capable_model.into()]),
(
Category::Any,
vec![efficient_model.into(), capable_model.into()],
),
]
.into(),
))
}
async fn run_classifier(
judge_model: &str,
efficient_model: &str,
capable_model: &str,
client: Arc<dyn RoutedLlmClient>,
request: Request,
) -> switchyard_libsy::Result<(ModelId, Response)> {
switchyard_llm_client::run(
classifier_router()?,
ClientRouter::single(client),
request,
classifier_models(judge_model, efficient_model, capable_model),
None,
)
.await
}
fn classifier_request() -> Request {
Request {
llm_request: text_request(Some("auto".to_string()), "classify this"),
raw_request: None,
metadata: None,
}
}
fn find_span(spans: &[SpanRecord], name: &str, field: &str, value: &str) -> SpanRecord {
match spans
.iter()
.find(|span| span.name == name && span.fields.get(field).map(String::as_str) == Some(value))
{
Some(span) => span.clone(),
None => panic!("no '{name}' span with {field}={value} in {spans:?}"),
}
}
fn find_otel_span(exporter: &InMemorySpanExporter, name: &str, model: &str) -> SpanData {
let spans = match exporter.get_finished_spans() {
Ok(spans) => spans,
Err(error) => panic!("failed to read exported spans: {error}"),
};
match spans.iter().find(|span| {
span.name == name
&& span.attributes.iter().any(|attribute| {
attribute.key.as_str() == "gen_ai.request.model"
&& attribute.value.as_str() == model
})
}) {
Some(span) => span.clone(),
None => {
let available = spans
.iter()
.map(|span| {
let model = span
.attributes
.iter()
.find(|attribute| attribute.key.as_str() == "gen_ai.request.model")
.map(|attribute| attribute.value.as_str().into_owned());
(span.name.to_string(), model)
})
.collect::<Vec<_>>();
panic!("no exported '{name}' span for model {model}; available: {available:?}")
}
}
}
fn otel_attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a OtelValue> {
span.attributes
.iter()
.find(|attribute| attribute.key.as_str() == key)
.map(|attribute| &attribute.value)
}
#[tokio::test]
async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
let event_count = store.events().len();
let router: Arc<dyn Algorithm> =
Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
config: TaskClassifierConfig {
classify_trigger: ClassifyTrigger::NewSession,
message_hash_fallback: true,
..TaskClassifierConfig::default()
},
})?);
let request = Request {
llm_request: LlmRequest {
messages: vec![Message::new(
Role::User,
vec![ContentBlock::Reasoning {
text: "provider reasoning".to_string(),
signature: None,
details: Vec::new(),
}],
)],
..LlmRequest::default()
},
raw_request: None,
metadata: None,
};
for _ in 0..2 {
let mut steps = router.clone().run_stream(
request.clone(),
classifier_models("warning/judge", "warning/efficient", "warning/capable"),
switchyard_libsy::State::default(),
);
let _ = steps.next().await;
}
let events = store.events();
let warnings = events[event_count..]
.iter()
.filter(|event| {
event.target == "libsy"
&& event.level == "WARN"
&& event
.fields
.get("message")
.is_some_and(|message| message.contains("affinity is enabled"))
})
.collect::<Vec<_>>();
assert_eq!(warnings.len(), 1, "affinity warnings: {warnings:?}");
assert!(warnings[0].fields.get("message").is_some_and(|message| {
message.contains("message-hash fallback with usable first-user text")
}));
Ok(())
}
#[tokio::test]
async fn stateful_escalation_warns_once_without_a_session_id() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
let event_count = store.events().len();
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
contract: ClassifierContractConfig::default(),
config: EscalationJudgeConfig::default(),
max_output_tokens: 64,
judge_deadline_ms: None,
})?) as Arc<dyn Algorithm>;
let client = Arc::new(JudgeClient {
judge_model: "warning-judge".into(),
outcome: JudgeOutcome::Reply(
r#"{"escalate":false,"category":"none","new_evidence":false,"reason":"progressing"}"#,
),
}) as Arc<dyn RoutedLlmClient>;
for _ in 0..2 {
switchyard_llm_client::run(
router.clone(),
ClientRouter::single(client.clone()),
classifier_request(),
classifier_models("warning-judge", "warning-efficient", "warning-capable"),
None,
)
.await?;
}
let warnings = store.events()[event_count..]
.iter()
.filter(|event| {
event.target == "libsy"
&& event.level == "WARN"
&& event.fields.get("message").is_some_and(|message| {
message.contains("stateful escalation has no session ID")
})
})
.count();
assert_eq!(warnings, 1);
Ok(())
}
#[tokio::test]
async fn deescalation_evidence_stays_pending_until_confirmed() -> switchyard_libsy::Result<()> {
// Shares the process-wide request gauge with the metrics tests, so take the same guard.
let _guard = serialize_test().lock().await;
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
contract: ClassifierContractConfig::default(),
config: EscalationJudgeConfig {
confirmations: 1,
deescalation: Some(DeescalationConfig {
strong_min_calls: 1,
strong_max_calls: None,
confirmations: 2,
weak_cooldown_calls: 0,
}),
..EscalationJudgeConfig::default()
},
max_output_tokens: 64,
judge_deadline_ms: None,
})?) as Arc<dyn Algorithm>;
let client = |verdict| {
Arc::new(JudgeClient {
judge_model: "evidence-judge".into(),
outcome: JudgeOutcome::Reply(verdict),
}) as Arc<dyn RoutedLlmClient>
};
let request = request_with_metadata("evidence-session", "evidence-correlation");
switchyard_llm_client::run(
router.clone(),
ClientRouter::single(client(
r#"{"escalate":true,"category":"repetition","new_evidence":true,"reason":"stuck"}"#,
)),
request.clone(),
classifier_models("evidence-judge", "evidence-efficient", "evidence-capable"),
None,
)
.await?;
let outcome = switchyard_llm_client::decide(
router,
ClientRouter::single(client(
r#"{"escalate":false,"category":"none","new_evidence":false,"reason":"recovered"}"#,
)),
request,
classifier_models("evidence-judge", "evidence-efficient", "evidence-capable"),
)
.await?;
assert_eq!(outcome.selected_model_id()?.as_str(), "evidence-capable");
assert_eq!(
outcome.metadata.and_then(|metadata| metadata.evidence),
Some(json!({"source": "escalation", "verdict": "pending"}))
);
Ok(())
}
#[tokio::test]
async fn affinity_keeps_the_algorithm_selection_after_client_fallback()
-> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (_, exporter, provider, _, _) = telemetry();
let before = flushed_metrics(exporter, provider);
let client = Arc::new(AffinityFallbackClient {
calls: Mutex::new(Vec::new()),
efficient_available: AtomicBool::new(false),
});
let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
config: TaskClassifierConfig {
judge: CapabilityJudgeConfig::Llm(LlmCapabilityConfig {
base_threshold: 0.5,
..LlmCapabilityConfig::default()
}),
classify_trigger: ClassifyTrigger::NewSession,
..TaskClassifierConfig::default()
},
})?) as Arc<dyn Algorithm>;
let request = request_with_metadata("affinity-fallback-session", "affinity-fallback-first");
let (selected, first_response) = switchyard_llm_client::run(
Arc::clone(&router),
ClientRouter::single(client.clone()),
request.clone(),
classifier_models(
"affinity-fallback-judge",
"affinity-fallback-weak",
"affinity-fallback-strong",
),
None,
)
.await?;
assert_eq!(selected, "affinity-fallback-weak");
assert_eq!(
first_response.served_model().map(ModelId::as_str),
Some("affinity-fallback-strong")
);
// One client request makes two routed calls: a weak failure and a strong success.
let after = flushed_metrics(exporter, provider);
for (metric, model, expected) in [
("switchyard.errors", "affinity-fallback-weak", 1),
("switchyard.requests", "affinity-fallback-weak", 0),
("switchyard.requests", "affinity-fallback-strong", 1),
] {
let attrs = [("model", model)];
assert_eq!(
u64_counter_value(&after, metric, &attrs).unwrap_or_default()
- u64_counter_value(&before, metric, &attrs).unwrap_or_default(),
expected,
"{metric} for {model}"
);
}
for (metric, expected) in [
("switchyard.total_requests", 2),
("switchyard.total_errors", 1),
] {
assert_eq!(
u64_gauge_value(&after, metric).unwrap_or_default()
- u64_gauge_value(&before, metric).unwrap_or_default(),
expected,
"{metric}"
);
}
for (model, outcome) in [
("affinity-fallback-weak", "error"),
("affinity-fallback-strong", "ok"),
] {
assert_eq!(
u64_counter_value(
&after,
"switchyard.llm_calls",
&[
("algorithm", "llm_task_classifier"),
("selected_model", model),
("outcome", outcome)
]
),
Some(1)
);
}
client.efficient_available.store(true, Ordering::Relaxed);
let (selected, second_response) = switchyard_llm_client::run(
router,
ClientRouter::single(client.clone()),
request,
classifier_models(
"affinity-fallback-judge",
"affinity-fallback-weak",
"affinity-fallback-strong",
),
None,
)
.await?;
assert_eq!(selected, "affinity-fallback-weak");
assert_eq!(
second_response.served_model().map(ModelId::as_str),
Some("affinity-fallback-weak")
);
assert_eq!(
&*client.calls.lock(),
&[
ModelId::from("affinity-fallback-judge"),
ModelId::from("affinity-fallback-weak"),
ModelId::from("affinity-fallback-strong"),
ModelId::from("affinity-fallback-weak"),
]
);
Ok(())
}
#[tokio::test]
async fn successful_run_records_metrics_spans_and_outcome_metadata() -> switchyard_libsy::Result<()>
{
let _guard = serialize_test().lock().await;
let (store, exporter, provider, span_exporter, _) = telemetry();
const ALGO: &str = "obs-success-algo";
const MODEL: &str = "obs-success-model";
let before = flushed_metrics(exporter, provider);
let total_requests_before =
u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();
let total_errors_before =
u64_gauge_value(&before, "switchyard.total_errors").unwrap_or_default();
let mut usage = Usage {
input_tokens: Some(11),
output_tokens: Some(7),
total_tokens: Some(25),
reasoning_tokens: Some(2),
cache: Usage::cache_details(Some(3), Some(4)),
provider_cost: None,
};
usage
.cache
.as_mut()
.expect("cache details")
.cache_creation_1h_input_tokens = Some(2);
let client = Arc::new(UsageClient { usage }) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-session-1", "obs-corr-1");
request.llm_request.sampling.temperature = Some(0.25);
request.llm_request.sampling.top_p = Some(0.9);
request.llm_request.sampling.top_k = Some(40);
request.llm_request.output.max_output_tokens = Some(512);
request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));
request.llm_request.reasoning.effort = Some("high".to_string());
let metadata = switchyard_libsy::OutcomeMetadata::new(
ALGO.to_string(),
Some(json!({
"source": "llm-classifier", "score": 0.9, "threshold": 0.5,
"capable_score": 0.7, "capable_threshold": 0.6,
"verdict": "continue", "trigger": "turn", "reason_code": "test",
"confidence": "wrong type", "unknown": LEAKED_CONTENT,
})),
);
let algorithm = Arc::new(SingleCallAlgo {
name: ALGO.to_string(),
target_set: vec![MODEL.to_string(), "obs-fallback-model".to_string()],
metadata: Some(metadata.clone()),
});
let (selected_model, _response) = run(algorithm, client, request).await?;
assert_eq!(selected_model, MODEL);
// Metrics: run/call counters and latency histograms keyed by algorithm,
// plus one published decision.
let snapshots = flushed_metrics(exporter, provider);
let run_attrs = [("algorithm", ALGO), ("outcome", "ok")];
let call_attrs = [
("algorithm", ALGO),
("selected_model", MODEL),
("outcome", "ok"),
];
let token_attrs = [("algorithm", ALGO), ("selected_model", MODEL)];
assert_eq!(
u64_counter_value(&snapshots, "switchyard.runs", &run_attrs),
Some(1)
);
assert_eq!(
f64_histogram_count(&snapshots, "switchyard.run_duration_ms", &run_attrs),
Some(1)
);
assert_eq!(
u64_counter_value(&snapshots, "switchyard.llm_calls", &call_attrs),
Some(1)
);
assert_eq!(
f64_histogram_count(&snapshots, "switchyard.llm_call_duration_ms", &call_attrs),
Some(1)
);
assert_eq!(
u64_counter_value(&snapshots, "switchyard.decisions", &token_attrs),
Some(1)
);
let routed_attrs = [("model", MODEL)];
assert_eq!(
u64_counter_value(&snapshots, "switchyard.requests", &routed_attrs),
Some(1)
);
assert_eq!(
f64_histogram_count(
&snapshots,
"switchyard.model_call_latency_ms",
&routed_attrs
),
Some(1)
);
assert_eq!(
u64_gauge_value(&snapshots, "switchyard.total_requests"),
Some(total_requests_before + 1)
);
assert_eq!(
u64_gauge_value(&snapshots, "switchyard.total_errors"),
Some(total_errors_before)
);
// One overhead observation per run, keyed by algorithm alone.
assert_eq!(
f64_histogram_count(
&snapshots,
"switchyard.routing_overhead_ms",
&[("algorithm", ALGO)]
),
Some(1)
);
// Spans: one run span carrying the correlation ids and outcome, one child
// llm_call span carrying the selection, outcome, and token counts.
let spans = store.spans();
let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO);
assert_eq!(
spans
.iter()
.filter(|span| span.name == "libsy.run"
&& span.fields.get("algorithm").map(String::as_str) == Some(ALGO))
.count(),
1
);
for (field, expected) in [
("outcome_id", metadata.outcome_id()),
("evidence.source", "llm-classifier"),
("evidence.score", "0.9"),
("evidence.threshold", "0.5"),
("evidence.capable_score", "0.7"),
("evidence.capable_threshold", "0.6"),
("evidence.verdict", "continue"),
("evidence.trigger", "turn"),
("evidence.reason_code", "test"),
] {
assert_eq!(
run_span.fields.get(field).map(String::as_str),
Some(expected),
"{field}"
);
}
assert!(!run_span.fields.contains_key("evidence.confidence"));
assert!(!format!("{run_span:?}").contains(LEAKED_CONTENT));
let exported = span_exporter.get_finished_spans().expect("exported spans");
let exported_run = exported
.iter()
.find(|span| {
span.name == "libsy.run"
&& otel_attribute(span, "outcome_id")
.is_some_and(|id| id.as_str() == metadata.outcome_id())
})
.expect("outcome span exported");
assert_eq!(
otel_attribute(exported_run, "evidence.score"),
Some(&OtelValue::F64(0.9))
);
assert_eq!(
otel_attribute(exported_run, "selected_model_ids"),
Some(&OtelValue::Array(OtelArray::String(vec![
MODEL.into(),
"obs-fallback-model".into()
]),))
);
assert_eq!(run_span.parent, None);
assert_eq!(
run_span.fields.get("session_id").map(String::as_str),
Some("obs-session-1")
);
assert_eq!(
run_span.fields.get("session.id").map(String::as_str),
Some("obs-session-1")
);
assert_eq!(
run_span
.fields
.get("switchyard.algorithm")
.map(String::as_str),
Some(ALGO)
);
assert_eq!(
run_span.fields.get("switchyard.route").map(String::as_str),
Some("auto")
);
assert_eq!(
run_span.fields.get("correlation_id").map(String::as_str),
Some("obs-corr-1")
);
assert_eq!(
run_span.fields.get("outcome").map(String::as_str),
Some("ok")
);
// Semantic harness labels: bounded classes, distinct from the high-cardinality ids above.
assert_eq!(
run_span.fields.get("task_kind").map(String::as_str),
Some("code_review")
);
assert_eq!(
run_span.fields.get("agent_role").map(String::as_str),
Some("reviewer")
);
assert!(!run_span.fields.contains_key("extra_metadata"));
// The default-client serve inside `run` gets its own client-call span.
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
// Host-side span: `run`'s serve loop creates it outside the algorithm's
// spans, so it has no libsy parent. This pins the instrument idiom
// (`#[tracing::instrument]` / `Future::instrument`) — an `Entered` guard
// held across the offload `.await` would leave `libsy.llm_call` entered
// on the thread and leak it in as the parent.
assert_eq!(client_span.parent.as_deref(), None);
assert_eq!(
client_span.fields.get("algorithm").map(String::as_str),
Some(ALGO)
);
assert_eq!(
client_span.fields.get("outcome").map(String::as_str),
Some("ok")
);
for (field, value) in [
("otel.kind", "client"),
("switchyard.algorithm", ALGO),
("otel.name", "chat obs-success-model"),
("gen_ai.operation.name", "chat"),
("gen_ai.request.model", MODEL),
("gen_ai.request.temperature", "0.25"),
("gen_ai.request.top_p", "0.9"),
("gen_ai.request.top_k", "40"),
("gen_ai.request.max_tokens", "512"),
("gen_ai.request.reasoning.level", "high"),
("gen_ai.output.type", "json"),
("gen_ai.conversation.id", "obs-session-1"),
("gen_ai.response.id", "obs-response-1"),
("gen_ai.response.model", MODEL),
("gen_ai.usage.input_tokens", "18"),
("gen_ai.usage.output_tokens", "7"),
("gen_ai.usage.cache_read.input_tokens", "3"),
("gen_ai.usage.cache_creation.input_tokens", "4"),
("gen_ai.usage.cache_creation_1h.input_tokens", "2"),
("gen_ai.usage.reasoning.output_tokens", "2"),
] {
assert_eq!(
client_span.fields.get(field).map(String::as_str),
Some(value),
"unexpected {field}"
);
}
assert_eq!(client_span.fields.get("gen_ai.request.stream"), None);
let otel_span = find_otel_span(span_exporter, "chat obs-success-model", MODEL);
assert!(matches!(
otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),
Some(OtelValue::Array(OtelArray::String(reasons)))
if reasons.len() == 1 && reasons[0].as_str() == "end_turn"
));
assert_eq!(
otel_attribute(&otel_span, "gen_ai.request.max_tokens"),
Some(&OtelValue::I64(512))
);
assert_eq!(
otel_attribute(&otel_span, "gen_ai.usage.input_tokens"),
Some(&OtelValue::I64(18))
);
// The algorithm logs why it made the decision.
let events = store.events();
assert!(
!events.iter().any(
|event| event.fields.get("algorithm").map(String::as_str) == Some(ALGO)
&& event.fields.get("message").map(String::as_str) == Some("routing decision")
)
);
assert!(
events.iter().any(|event| {
event.level == "INFO"
&& event
.fields
.get("message")
.is_some_and(|message| message.contains("picked"))
}),
"no routing-decision log event for {MODEL} in {events:?}"
);
Ok(())
}
#[tokio::test]
async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (_, exporter, provider, _, _) = telemetry();
const STRONG: &str = "obs-stage-strong";
const WEAK: &str = "obs-stage-weak";
let algorithm = Arc::new(StageRouter::new(StageRouterConfig::new(
PickerMode::EfficientFirst,
0.5,
))?) as Arc<dyn Algorithm>;
let request = Request {
llm_request: LlmRequest {
model: Some("auto".to_string()),
messages: vec![
Message::text(Role::User, "fix the build"),
Message::new(
Role::Assistant,
vec![ContentBlock::ToolCall(ToolCall {
id: "call_1".to_string(),
name: "Bash".to_string(),
arguments: json!({"command": "cargo test"}),
})],
),
Message::new(
Role::Tool,
vec![ContentBlock::ToolResult(ToolResult {
tool_call_id: "call_1".to_string(),
content: vec![ContentBlock::Text {
text: "fatal runtime error: out of memory".to_string(),
}],
is_error: Some(true),
})],
),
],
..LlmRequest::default()
},
raw_request: None,
metadata: Some(Metadata {
wire_format: Some(WireFormat::OpenAiChat),
session_id: Some("obs-stage-session".to_string()),
..Metadata::default()
}),
};
let client = Arc::new(UsageClient {
usage: Usage::default(),
}) as Arc<dyn RoutedLlmClient>;
let (selected_model, _) = switchyard_llm_client::run(
algorithm,
ClientRouter::single(client),
request,
Arc::new(RuntimeModels::new(
[
(Category::Capable, vec![ModelId::from(STRONG)]),
(Category::Efficient, vec![ModelId::from(WEAK)]),
(
Category::Any,
vec![ModelId::from(STRONG), ModelId::from(WEAK)],
),
]
.into(),
)),
None,
)
.await?;
assert_eq!(selected_model, STRONG);
let snapshots = flushed_metrics(exporter, provider);
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.stage_router.routing_decisions",
&[("decision_source", "override"), ("target_name", STRONG)],
),
Some(1)
);
for name in [
"switchyard.stage_router.probability",
"switchyard.stage_router.confidence",
"switchyard.stage_router.severity",
"switchyard.stage_router.spinning",
"switchyard.stage_router.exploring",
"switchyard.stage_router.production_intensity",
] {
assert_eq!(f64_histogram_count(&snapshots, name, &[]), Some(1));
}
Ok(())
}
#[tokio::test]
async fn observed_run_reports_one_successful_routed_call() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let observations = Arc::new(Mutex::new(Vec::new()));
let observed = Arc::clone(&observations);
let observer: RunObserver = Arc::new(move |observation| observed.lock().push(observation));
const ALGO: &str = "observed-run-algo";
const MODEL: &str = "observed-run-model";
let client = Arc::new(UsageClient {
usage: Usage::default(),
}) as Arc<dyn RoutedLlmClient>;
let (_, response) = switchyard_llm_client::run(
algo(ALGO, MODEL),
ClientRouter::single(client),
request_with_metadata("observed-session", "observed-correlation"),
Arc::new(RuntimeModels::default()),
Some(observer),
)
.await?;
assert_eq!(
response
.llm_response
.as_agg()
.map(|response| response.model.as_deref()),
Some(Some(MODEL))
);
let observations = observations.lock();
// Outcome metadata precedes the answer call and final overhead observation.
assert_eq!(observations.len(), 3);
let RunObservation::Outcome(metadata) = &observations[0] else {
return Err(test_error("expected an outcome observation"));
};
assert_eq!(metadata.algorithm, ALGO);
let RunObservation::AnswerCall(observation) = &observations[1] else {
return Err(test_error("expected an answer-call observation"));
};
assert_eq!(observation.selected_model, MODEL);
assert!(observation.is_success);
assert!(observation.usage.is_some());
assert!(matches!(
observations[2],
RunObservation::RoutingOverhead(_)
));
Ok(())
}
#[tokio::test]
async fn answered_routing_call_keeps_its_execution_phase() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
const ALGO: &str = "answered-routing-call";
const MODEL: &str = "answer-from-routing";
switchyard_llm_client::run(
Arc::new(RoutingCallAlgo {
name: ALGO.to_string(),
target: MODEL.into(),
}),
ClientRouter::single(Arc::new(UsageClient {
usage: Usage::default(),
})),
request_with_metadata("answered-routing-session", "answered-routing-correlation"),
Arc::new(RuntimeModels::default()),
None,
)
.await?;
let span = find_span(&store.spans(), "libsy.client_call", "selected_model", MODEL);
assert_eq!(
span.fields.get("switchyard.call_phase").map(String::as_str),
Some("routing")
);
Ok(())
}
/// A streamed response keeps the client span available until terminal usage arrives.
struct StreamingUsageClient;
#[async_trait]
impl RoutedLlmClient for StreamingUsageClient {
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
let engine = switchyard_translation::TranslationEngine::default();
let mut state = switchyard_translation::StreamTranslationState::new(
WireFormat::OpenAiChat,
WireFormat::OpenAiChat,
);
let chunks = [
json!({"id": "", "model": request.model_id(), "choices": []}),
json!({"id": "", "choices": [{"index": 0, "delta": {"content": "hello"}}]}),
json!({"id": "obs-stream-response", "choices": []}),
json!({"id": "obs-stream-response", "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 21, "completion_tokens": 5, "prompt_tokens_details": {"cached_tokens": 8}}}),
].into_iter().map(|event| {
Ok(engine.decode_stream_event(&mut state, WireFormat::OpenAiChat, event)
.expect("valid Chat fixture"))
}).collect::<Vec<_>>();
Ok(Response {
llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
}
}
#[tokio::test]
async fn buffered_routing_closes_observation_before_replay() -> switchyard_libsy::Result<()> {
use tracing::Instrument;
let _guard = serialize_test().lock().await;
let (_, _, _, exporter, _) = telemetry();
for drain in [false, true] {
let model = if drain {
"routing-replay-drained"
} else {
"routing-replay-dropped"
};
let mut request = request_with_metadata(model, model);
request.llm_request.stream = true;
let (_, response) = run(
Arc::new(RoutingCallAlgo {
name: model.into(),
target: model.into(),
}),
Arc::new(StreamingUsageClient),
request,
)
.instrument(tracing::info_span!(
"buffered-workflow",
gen_ai.request.model = model
))
.await?;
// Upstream I/O is complete, so usage and parentage must export before replay.
let call = find_otel_span(exporter, &format!("chat {model}"), model);
let workflow = find_otel_span(exporter, "buffered-workflow", model);
assert_eq!(call.parent_span_id, workflow.span_context.span_id());
for (field, tokens) in [
("gen_ai.usage.input_tokens", 21),
("gen_ai.usage.output_tokens", 5),
] {
assert_eq!(otel_attribute(&call, field), Some(&OtelValue::I64(tokens)));
}
let count = exporter.get_finished_spans().expect("exported spans").len();
if drain {
let replay = response
.llm_response
.into_agg()
.await
.expect("valid replay");
assert_eq!(
replay.outputs[0].content,
vec![ContentBlock::Text {
text: "hello".into()
}]
);
} else {
drop(response);
}
assert_eq!(
exporter.get_finished_spans().expect("exported spans").len(),
count,
"consuming or dropping replay must not export another observation"
);
}
Ok(())
}
struct FailingStreamClient(bool);
#[async_trait]
impl RoutedLlmClient for FailingStreamClient {
async fn call(&self, _request: Request) -> Result<Response, LlmClientError> {
let failure = if self.0 {
Err(LlmClientError::UpstreamHttp {
status: http::StatusCode::BAD_GATEWAY,
body: LEAKED_CONTENT.into(),
})
} else {
Ok(LlmResponseChunk::StreamError {
message: LEAKED_CONTENT.into(),
}
.into())
};
Ok(Response {
llm_response: LlmResponse::Stream(
futures::stream::iter([
Ok(LlmResponseChunk::Usage(Usage {
input_tokens: Some(11),
output_tokens: Some(3),
..Usage::default()
})
.into()),
failure,
])
.boxed(),
),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})
}
}
#[tokio::test]
async fn routing_stream_error_is_recorded_once_without_upstream_body() {
let _guard = serialize_test().lock().await;
let (_, _, _, exporter, _) = telemetry();
for http_error in [false, true] {
let model = if http_error {
"routing-http-stream-error"
} else {
"routing-in-band-error"
};
let mut request = request_with_metadata(model, model);
request.llm_request.stream = true;
assert!(
run(
Arc::new(RoutingCallAlgo {
name: model.into(),
target: model.into()
}),
Arc::new(FailingStreamClient(http_error)),
request,
)
.await
.is_err()
);
let call = find_otel_span(exporter, &format!("chat {model}"), model);
for (key, expected) in [
("outcome", OtelValue::String("error".into())),
("error.type", OtelValue::String("502".into())),
("error", OtelValue::String("upstream HTTP 502".into())),
("gen_ai.usage.input_tokens", OtelValue::I64(11)),
("gen_ai.usage.output_tokens", OtelValue::I64(3)),
] {
let values: Vec<_> = call
.attributes
.iter()
.filter(|attr| attr.key.as_str() == key)
.map(|attr| attr.value.clone())
.collect();
assert_eq!(
values,
vec![expected],
"{key} must be exported exactly once"
);
}
assert!(!format!("{:?}", call.attributes).contains(LEAKED_CONTENT));
}
}
struct TimeoutClient;
#[async_trait]
impl RoutedLlmClient for TimeoutClient {
async fn call(&self, _request: Request) -> Result<Response, LlmClientError> {
Err(LlmClientError::Timeout {
source: Box::new(TestError("upstream timed out")),
})
}
}
struct HostErrorClient;
#[async_trait]
impl RoutedLlmClient for HostErrorClient {
async fn call(&self, _request: Request) -> Result<Response, LlmClientError> {
Err(LlmClientError::Host {
source: Box::new(TestError(LEAKED_CONTENT)),
})
}
}
#[tokio::test]
async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, _, _, span_exporter, _) = telemetry();
const ALGO: &str = "obs-stream-algo";
const MODEL: &str = "obs-stream-model";
let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-stream-session", "obs-stream-corr");
request.llm_request.stream = true;
let (_, response) = run(algo(ALGO, MODEL), client, request).await?;
let LlmResponse::Stream(mut stream) = response.llm_response else {
return Err(test_error("expected a streamed response"));
};
let engine = switchyard_translation::TranslationEngine::default();
let mut translated = switchyard_translation::StreamTranslationState::new(
WireFormat::OpenAiChat,
WireFormat::OpenAiResponses,
);
let mut events = Vec::new();
while let Some(item) = stream.next().await {
events.extend(
engine
.encode_stream_event(
&mut translated,
WireFormat::OpenAiResponses,
item.expect("valid stream"),
)
.expect("valid translation"),
);
}
events.extend(
engine
.finish_stream(&mut translated, WireFormat::OpenAiResponses)
.expect("valid finish"),
);
let created = events
.iter()
.find(|e| e["type"] == "response.created")
.expect("created");
let completed = events
.iter()
.find(|e| e["type"] == "response.completed")
.expect("completed");
assert_eq!(created["response"]["id"], completed["response"]["id"]);
assert_eq!(
completed["response"]["output"][0]["content"][0]["text"],
"hello"
);
let spans = store.spans();
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
for (field, value) in [
("otel.name", "chat obs-stream-model"),
("gen_ai.request.stream", "true"),
("gen_ai.response.id", "obs-stream-response"),
("gen_ai.response.model", MODEL),
("gen_ai.usage.input_tokens", "21"),
("gen_ai.usage.output_tokens", "5"),
("gen_ai.usage.cache_read.input_tokens", "8"),
] {
assert_eq!(
client_span.fields.get(field).map(String::as_str),
Some(value),
"unexpected {field}"
);
}
let otel_span = find_otel_span(span_exporter, "chat obs-stream-model", MODEL);
assert!(matches!(
otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),
Some(OtelValue::Array(OtelArray::String(reasons)))
if reasons.len() == 1 && reasons[0].as_str() == "stop"
));
Ok(())
}
#[tokio::test]
async fn dropped_stream_records_cancelled_outcome() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
const ALGO: &str = "obs-cancelled-stream-algo";
const MODEL: &str = "obs-cancelled-stream-model";
let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;
let mut request = request_with_metadata("obs-cancelled-session", "obs-cancelled-corr");
request.llm_request.stream = true;
let (_, response) = run(algo(ALGO, MODEL), client, request).await?;
let LlmResponse::Stream(stream) = response.llm_response else {
return Err(test_error("expected a streamed response"));
};
drop(stream);
let spans = store.spans();
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
assert_eq!(
client_span.fields.get("outcome").map(String::as_str),
Some("cancelled")
);
Ok(())
}
#[tokio::test]
async fn typed_client_failure_records_semantic_error_type() {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
const ALGO: &str = "obs-timeout-algo";
const MODEL: &str = "obs-timeout-model";
let result = run(
algo(ALGO, MODEL),
Arc::new(TimeoutClient),
request_with_metadata("obs-timeout-session", "obs-timeout-corr"),
)
.await;
assert!(matches!(
result,
Err(LibsyError::ClientCall {
source: LlmClientError::Timeout { .. },
..
})
));
let spans = store.spans();
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
assert_eq!(
client_span.fields.get("error.type").map(String::as_str),
Some("timeout")
);
}
// Host error text must not appear in the client span.
#[tokio::test]
async fn host_error_source_is_redacted_from_the_client_call_span() {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
const MODEL: &str = "obs-host-error-model";
let _ = run(
algo("obs-host-error-algo", MODEL),
Arc::new(HostErrorClient),
request_with_metadata("obs-host-error-session", "obs-host-error-corr"),
)
.await;
let spans = store.spans();
let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
assert_eq!(
client_span.fields.get("error.type").map(String::as_str),
Some("host")
);
assert!(!format!("{client_span:?}").contains(LEAKED_CONTENT));
}
/// Verifies that the client span retains the HTTP status without the upstream body.
#[tokio::test]
async fn upstream_body_is_redacted_from_the_client_call_span() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, _, _, _, _) = telemetry();
const JUDGE: &str = "redaction-judge";
let client = Arc::new(JudgeClient {
judge_model: JUDGE.into(),
outcome: JudgeOutcome::CallFailure,
}) as Arc<dyn RoutedLlmClient>;
let result = run_classifier(
JUDGE,
"redaction-weak",
"redaction-strong",
client,
classifier_request(),
)
.await;
assert!(matches!(
result,
Err(LibsyError::ClientCall {
source: LlmClientError::UpstreamHttp { status, body },
..
}) if status == http::StatusCode::INTERNAL_SERVER_ERROR && body.contains(LEAKED_CONTENT)
));
let spans = store.spans();
let client_span = find_span(&spans, "libsy.client_call", "selected_model", JUDGE);
assert_eq!(
client_span.fields.get("error.type").map(String::as_str),
Some("500")
);
let error = client_span
.fields
.get("error")
.map(String::as_str)
.unwrap_or("");
assert!(error.contains("upstream HTTP 500"), "{client_span:?}");
assert!(!error.contains(LEAKED_CONTENT), "{client_span:?}");
assert!(!spans.iter().any(|span| {
span.name == "libsy.client_call"
&& matches!(
span.fields.get("selected_model").map(String::as_str),
Some("redaction-weak" | "redaction-strong")
)
}));
Ok(())
}
#[tokio::test]
async fn decision_calls_record_each_terminal_path_once() -> switchyard_libsy::Result<()> {
use switchyard_protocol::{DecisionRequest, DecisionResponse};
let _guard = serialize_test().lock().await;
let (store, exporter, provider, _, _) = telemetry();
struct DecisionAlgo(String);
#[async_trait]
impl Algorithm for DecisionAlgo {
fn name(&self) -> &str {
&self.0
}
async fn route(
self: Arc<Self>,
driver: Driver,
request: Request,
) -> switchyard_libsy::Result<RoutingOutcome> {
driver
.call_decision(
DecisionRequest {
model: None,
input: json!(LEAKED_CONTENT),
questions: Default::default(),
},
self.0.clone().into(),
)
.await?;
Ok(RoutingOutcome::route_to("answer".into(), vec![], request))
}
}
for mode in ["reply", "error", "fail", "drop", "cancel"] {
let algorithm = format!("obs-decision-{mode}");
let name = algorithm.as_str();
let mut stream = Box::pin(Arc::new(DecisionAlgo(algorithm.clone())).run_stream(
request_with_metadata("decision-session", "decision-correlation"),
Arc::new(RuntimeModels::default()),
switchyard_libsy::State::default(),
));
let Some(Ok(Step::CallDecision(call))) = stream.next().await else {
return Err(test_error("expected a decision call"));
};
assert_eq!(
u64_counter_value(
&flushed_metrics(exporter, provider),
"switchyard.decision_calls",
&[("algorithm", name)],
),
None,
);
tokio::time::sleep(Duration::from_millis(10)).await;
if mode == "cancel" {
drop(stream);
drop(call);
} else {
match mode {
"reply" => {
call.respond(std::future::ready(Ok(DecisionResponse {
id: Some("decision-response".into()),
model: Some("provider-model".into()),
answers: Default::default(),
usage: Usage {
input_tokens: Some(42),
output_tokens: Some(5),
..Usage::default()
},
})))
.await?
}
"error" => {
call.respond(std::future::ready(Err(test_error(LEAKED_CONTENT))))
.await?
}
"fail" => assert!(call.fail(test_error(LEAKED_CONTENT)).is_err()),
"drop" => drop(call),
_ => unreachable!(),
}
while stream.next().await.is_some() {}
}
let outcome = if mode == "reply" { "ok" } else { "error" };
let attrs = [
("algorithm", name),
("selected_model", name),
("outcome", outcome),
];
let snapshots = flushed_metrics(exporter, provider);
assert_eq!(
u64_counter_value(&snapshots, "switchyard.decision_calls", &attrs),
Some(1)
);
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.decision_calls",
&[
("algorithm", name),
("outcome", if mode == "reply" { "error" } else { "ok" })
],
),
None,
);
assert_eq!(
f64_histogram_count(&snapshots, "switchyard.decision_call_duration_ms", &attrs),
Some(1)
);
assert!(
f64_histogram_sum_ms(&snapshots, "switchyard.decision_call_duration_ms", &attrs)
.unwrap_or(0)
>= 10
);
assert_eq!(
u64_counter_value(&snapshots, "switchyard.llm_calls", &[("algorithm", name)]),
None
);
let span = find_span(&store.spans(), "libsy.decision_call", "algorithm", name);
assert_eq!(span.parent.as_deref(), Some("libsy.run"));
assert_eq!(
span.fields.get("outcome").map(String::as_str),
Some(outcome)
);
for (field, value) in [
("input_tokens", "42"),
("output_tokens", "5"),
("gen_ai.response.id", "decision-response"),
("gen_ai.response.model", "provider-model"),
] {
assert_eq!(
span.fields.get(field).map(String::as_str),
(mode == "reply").then_some(value)
);
}
assert!(!span.fields.contains_key("total_tokens"));
assert!(!span.fields.contains_key("error"));
assert!(!format!("{span:?}").contains(LEAKED_CONTENT));
}
Ok(())
}
#[tokio::test]
async fn failed_call_records_metrics_without_error_details() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, exporter, provider, _, _) = telemetry();
const ALGO: &str = "obs-failure-algo";
const MODEL: &str = "obs-failure-model";
// The routing call is offloaded and we fail it by hand, without a client.
let algorithm = Arc::new(RoutingCallAlgo {
name: ALGO.to_string(),
target: MODEL.into(),
});
let stream = algorithm.run_stream(
request_with_metadata("obs-session-2", "obs-corr-2"),
Arc::new(RuntimeModels::default()),
switchyard_libsy::State::default(),
);
tokio::pin!(stream);
let mut saw_error_step = false;
while let Some(step) = stream.next().await {
match step {
Ok(Step::CallModel(mut call)) => {
call.recover_errors = true;
call.respond(std::future::ready(Err(test_error(
"synthetic upstream failure",
))))
.await?;
}
Ok(Step::CallDecision(_)) => return Err(test_error("unexpected decision call")),
Ok(Step::Done(_)) => {
return Err(test_error("expected the failed call to fail the run"));
}
Err(error) => {
assert!(error.to_string().contains("synthetic upstream failure"));
saw_error_step = true;
}
}
}
assert!(
saw_error_step,
"expected an error step from the failed call"
);
// The routing call and algorithm run fail, but no terminal outcome exists to attribute as a
// routed request.
let snapshots = flushed_metrics(exporter, provider);
let run_attrs = [("algorithm", ALGO), ("outcome", "error")];
let call_attrs = [
("algorithm", ALGO),
("selected_model", MODEL),
("outcome", "error"),
];
assert_eq!(
u64_counter_value(&snapshots, "switchyard.runs", &run_attrs),
Some(1)
);
assert_eq!(
u64_counter_value(&snapshots, "switchyard.llm_calls", &call_attrs),
Some(1)
);
// Nothing was served, so there is nothing to measure routing against.
assert_eq!(
f64_histogram_count(
&snapshots,
"switchyard.routing_overhead_ms",
&[("algorithm", ALGO)]
),
None
);
// Error details remain with the caller, while operational status is preserved.
let spans = store.spans();
let run_span = find_span(&spans, "libsy.run", "algorithm", ALGO);
assert_eq!(
run_span.fields.get("outcome").map(String::as_str),
Some("error")
);
assert!(!run_span.fields.contains_key("error"));
assert!(!run_span.fields.contains_key("outcome_id"));
let call_span = find_span(&spans, "libsy.llm_call", "selected_model", MODEL);
assert_eq!(
call_span.fields.get("outcome").map(String::as_str),
Some("error")
);
assert_eq!(call_span.parent.as_deref(), Some("libsy.run"));
assert!(!call_span.fields.contains_key("error"));
let events = store.events();
assert!(!events.iter().any(|event| event.target == "libsy"
&& event.fields.get("algorithm").map(String::as_str) == Some(ALGO)));
Ok(())
}
#[tokio::test]
async fn classifier_metrics_count_routing_and_answer_calls_once() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, exporter, provider, _, _) = telemetry();
let before = flushed_metrics(exporter, provider);
let total_requests_before =
u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();
let client = Arc::new(ClassifierClient {
classifier_model_id: "classifier".into(),
classifier_delay: Duration::from_millis(60),
routed_delay: Duration::from_millis(200),
}) as Arc<dyn RoutedLlmClient>;
let (selected_model, _response) =
run_classifier("classifier", "weak", "strong", client, classifier_request()).await?;
assert_eq!(selected_model, "weak");
let spans = store.spans();
let span = find_span(&spans, "libsy.client_call", "selected_model", "weak");
assert_eq!(
span.fields.get("switchyard.call_phase").map(String::as_str),
Some("completion")
);
let snapshots = flushed_metrics(exporter, provider);
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.llm_calls",
&[
("algorithm", "llm_task_classifier"),
("selected_model", "classifier"),
("outcome", "ok"),
],
),
Some(1)
);
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.llm_calls",
&[
("algorithm", "llm_task_classifier"),
("selected_model", "weak"),
("outcome", "ok"),
],
),
Some(1)
);
assert_eq!(
u64_counter_value(&snapshots, "switchyard.requests", &[("model", "weak")],),
Some(1)
);
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.requests",
&[("model", "classifier")],
),
None
);
assert_eq!(
f64_histogram_count(
&snapshots,
"switchyard.model_call_latency_ms",
&[("model", "classifier")],
),
None
);
assert_eq!(
u64_gauge_value(&snapshots, "switchyard.total_requests"),
Some(total_requests_before + 1)
);
// The classifier call is the router's own work but the routed call is not,
// so overhead lands near the classifier's 60ms, not their 260ms sum.
let overhead = f64_histogram_sum_ms(
&snapshots,
"switchyard.routing_overhead_ms",
&[("algorithm", "llm_task_classifier")],
)
.unwrap_or_default();
assert!(
(60..200).contains(&overhead),
"expected roughly the classifier's 60ms, got {overhead}ms"
);
Ok(())
}
#[tokio::test]
async fn classifier_stops_on_client_errors_and_records_verdict_fallback()
-> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (_, exporter, provider, _, _) = telemetry();
let cases = [
("fo-call", JudgeOutcome::CallFailure, None),
(
"fo-parse",
JudgeOutcome::Reply("not json at all"),
Some("parse_error"),
),
("fo-stream-decode", JudgeOutcome::StreamDecodeFailure, None),
(
"fo-valid",
JudgeOutcome::Reply(
r#"{"crux":"hard task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.3}"#,
),
None,
),
];
for (judge_model, outcome, expected_reason) in cases {
let client = Arc::new(JudgeClient {
judge_model: judge_model.into(),
outcome,
});
let result = run_classifier(
judge_model,
"fo-weak",
"fo-strong",
client.clone(),
classifier_request(),
)
.await;
match client.outcome {
JudgeOutcome::CallFailure => assert!(result.is_err(), "{judge_model}"),
JudgeOutcome::StreamDecodeFailure => assert!(matches!(
result,
Err(LibsyError::ClientCall {
source: LlmClientError::ResponseTranslation { .. },
..
})
)),
JudgeOutcome::Reply(_) => assert_eq!(result?.0.as_str(), "fo-strong"),
}
let snapshots = flushed_metrics(exporter, provider);
let outcome = match client.outcome {
JudgeOutcome::CallFailure | JudgeOutcome::StreamDecodeFailure => "error",
JudgeOutcome::Reply(_) => "ok",
};
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.llm_calls",
&[
("algorithm", "llm_task_classifier"),
("selected_model", judge_model),
("outcome", outcome),
],
),
Some(1),
"logical call accounting for {judge_model}"
);
match expected_reason {
Some(reason) => assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.classifier_fail_open",
&[("reason", reason), ("judge_model", judge_model)],
),
Some(1),
"case {reason} did not count the fail-open"
),
None => assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.classifier_fail_open",
&[("judge_model", judge_model)],
),
None,
"client failures and valid verdicts must not increment switchyard.classifier_fail_open"
),
}
}
Ok(())
}
#[tokio::test]
async fn classifier_can_fail_open_on_judge_client_errors() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
assert!(TaskClassifierConfig::default().fail_open);
for (judge, reason) in [
(
Arc::new(TimeoutClient) as Arc<dyn RoutedLlmClient>,
"timeout",
),
(
Arc::new(JudgeClient {
judge_model: "judge".into(),
outcome: JudgeOutcome::CallFailure,
}),
"upstream_5xx",
),
(
Arc::new(JudgeClient {
judge_model: "judge".into(),
outcome: JudgeOutcome::StreamDecodeFailure,
}),
"invalid_response",
),
] {
let algorithm = |fail_open: Option<bool>| -> switchyard_libsy::Result<Arc<dyn Algorithm>> {
let mut config = serde_json::json!({"base_threshold": 0.5});
if let Some(fail_open) = fail_open {
config["fail_open"] = fail_open.into();
}
Ok(Arc::new(LlmTaskClassifier::new(
LlmClassifierConfig::Capability {
config: serde_json::from_value(config).unwrap(),
},
)?))
};
let clients = ClientRouter::new(std::collections::HashMap::from([
(ModelId::from("judge"), judge),
(
ModelId::from("capable"),
Arc::new(JudgeClient {
judge_model: "judge".into(),
outcome: JudgeOutcome::CallFailure,
}) as Arc<dyn RoutedLlmClient>,
),
]));
let models = classifier_models("judge", "efficient", "capable");
let stopped = switchyard_llm_client::decide(
algorithm(Some(false))?,
clients.clone(),
classifier_request(),
models.clone(),
)
.await;
assert!(
matches!(stopped, Err(LibsyError::ClientCall { .. })),
"{reason}"
);
let outcome = switchyard_llm_client::decide(
algorithm(None)?,
clients.clone(),
classifier_request(),
models.clone(),
)
.await?;
assert_eq!(outcome.selected_model_id()?.as_str(), "capable");
assert_eq!(
outcome.metadata.and_then(|metadata| metadata.evidence),
Some(serde_json::json!({"source": "fail_open", "reason_code": reason})),
);
let (selected, _) = switchyard_llm_client::run(
algorithm(Some(true))?,
clients,
classifier_request(),
models.clone(),
None,
)
.await?;
assert_eq!(selected.as_str(), "capable");
let answer_failure = switchyard_llm_client::run(
algorithm(None)?,
ClientRouter::single(Arc::new(TimeoutClient)),
classifier_request(),
models,
None,
)
.await;
assert!(matches!(answer_failure, Err(LibsyError::ClientCall {
target, source: LlmClientError::Timeout { .. },
}) if target.as_str() == "capable"));
}
Ok(())
}
#[tokio::test]
async fn in_flight_gauge_reads_a_run_parked_on_an_unanswered_routing_call()
-> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (_, exporter, provider, _, _) = telemetry();
const ALGO: &str = "obs-in-flight-algo";
const MODEL: &str = "obs-in-flight-model";
let algorithm = Arc::new(RoutingCallAlgo {
name: ALGO.to_string(),
target: MODEL.into(),
});
let stream = algorithm.run_stream(
request_with_metadata("obs-session-if", "obs-corr-if"),
Arc::new(RuntimeModels::default()),
switchyard_libsy::State::default(),
);
tokio::pin!(stream);
let attributes = [("algorithm", ALGO)];
// Take the offloaded call and hold it without responding. This is the shape of a
// stalled classifier: the run has started and cannot proceed until the call returns.
let Some(Ok(Step::CallModel(call))) = stream.next().await else {
return Err(test_error("expected an offloaded routing call"));
};
let snapshots = flushed_metrics(exporter, provider);
assert_eq!(
i64_up_down_counter_value(&snapshots, "switchyard.algorithms_in_flight", &attributes),
Some(1),
"a run waiting on an unanswered routing call must read as in flight"
);
// The gap this gauge fills: every other run metric is recorded on resolution, so
// while the call is outstanding they say nothing at all.
assert_eq!(
u64_counter_value(&snapshots, "switchyard.runs", &attributes),
None,
"the run counter must not report a run that has not resolved"
);
call.respond(std::future::ready(Ok(Response {
llm_response: LlmResponse::Agg(text_response(Some(MODEL.to_string()), "answer")),
metadata: None,
upstream_headers: http::HeaderMap::new(),
})))
.await?;
while stream.next().await.is_some() {}
let snapshots = flushed_metrics(exporter, provider);
assert_eq!(
i64_up_down_counter_value(&snapshots, "switchyard.algorithms_in_flight", &attributes),
Some(0),
"the gauge must fall back to zero once the run resolves"
);
Ok(())
}
#[tokio::test]
async fn in_flight_gauge_clears_when_a_run_is_abandoned() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, exporter, provider, _, _) = telemetry();
const ALGO: &str = "obs-abandoned-algo";
const MODEL: &str = "obs-abandoned-model";
let algorithm = Arc::new(RoutingCallAlgo {
name: ALGO.to_string(),
target: MODEL.into(),
});
let attributes = [("algorithm", ALGO)];
// Drop the step stream while the routing call is still outstanding, the way a
// disconnected client abandons a run. The run task is aborted mid-await and never
// reaches the code that follows it, so only a drop can return the count.
{
let stream = algorithm.run_stream(
request_with_metadata("obs-session-ab", "obs-corr-ab"),
Arc::new(RuntimeModels::default()),
switchyard_libsy::State::default(),
);
tokio::pin!(stream);
let Some(Ok(Step::CallModel(_call))) = stream.next().await else {
return Err(test_error("expected an offloaded routing call"));
};
assert_eq!(
i64_up_down_counter_value(
&flushed_metrics(exporter, provider),
"switchyard.algorithms_in_flight",
&attributes,
),
Some(1),
);
}
// Let the runtime finish aborting the task it was told to drop.
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(
i64_up_down_counter_value(
&flushed_metrics(exporter, provider),
"switchyard.algorithms_in_flight",
&attributes,
),
Some(0),
"an abandoned run must not strand the gauge above zero"
);
let call_span = find_span(&store.spans(), "libsy.llm_call", "selected_model", MODEL);
assert_eq!(
call_span.fields.get("outcome").map(String::as_str),
Some("error")
);
Ok(())
}
/// Routing keys remain distinct while both routing-time and answer metrics aggregate by provider.
#[tokio::test]
async fn target_aliases_preserve_provider_model_metric_labels() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
let (store, exporter, provider, span_exporter, _) = telemetry();
struct MappedClient {
model: ModelId,
}
#[async_trait]
impl RoutedLlmClient for MappedClient {
fn provider_model<'a>(&'a self, _: &'a ModelId) -> &'a ModelId {
&self.model
}
async fn call(&self, request: Request) -> Result<Response, LlmClientError> {
UsageClient {
usage: Usage::default(),
}
.call(request)
.await
}
}
for routing_time in [false, true] {
let name = if routing_time {
"mapped-routing"
} else {
"mapped-answer"
};
let provider_model = ModelId::from(format!("provider/{name}"));
let client = Arc::new(MappedClient {
model: provider_model.clone(),
});
let models = Arc::new(
RuntimeModels::default().with_provider_models(
["efficient", "capable"]
.into_iter()
.map(|target| (ModelId::from(target), provider_model.clone()))
.chain([("fallback".into(), "provider/fallback".into())])
.collect(),
),
);
for target in ["efficient", "capable"] {
let algorithm = if routing_time {
Arc::new(RoutingCallAlgo {
name: name.into(),
target: target.into(),
}) as Arc<dyn Algorithm>
} else {
Arc::new(SingleCallAlgo {
name: name.into(),
target_set: vec![target.into(), "fallback".into(), "provider/direct".into()],
metadata: None,
})
};
let (selected, _) = switchyard_llm_client::run(
algorithm,
ClientRouter::single(client.clone()),
Request {
llm_request: text_request(None, "hello"),
..Request::default()
},
models.clone(),
None,
)
.await?;
assert_eq!(selected, target);
let spans = store
.spans()
.into_iter()
.rev()
.filter(|span| span.fields.get("algorithm").map(String::as_str) == Some(name))
.collect::<Vec<_>>();
let span = find_span(&spans, "libsy.client_call", "selected_target", target);
assert_eq!(span.fields["selected_model"], provider_model.as_str());
assert_eq!(span.fields["gen_ai.request.model"], provider_model.as_str());
assert_eq!(span.fields["otel.name"], format!("chat {provider_model}"));
let run_span = find_span(&spans, "libsy.run", "algorithm", name);
let exported = span_exporter.get_finished_spans().expect("exported spans");
let exported_run = exported
.iter()
.find(|span| {
span.name == "libsy.run"
&& otel_attribute(span, "outcome_id")
.is_some_and(|id| id.as_str() == run_span.fields["outcome_id"])
})
.expect("outcome span exported");
let mut expected_models = vec![provider_model.to_string().into()];
if !routing_time {
expected_models.extend(["provider/fallback".into(), "provider/direct".into()]);
}
assert_eq!(
otel_attribute(exported_run, "selected_model_ids"),
Some(&OtelValue::Array(OtelArray::String(expected_models)))
);
}
let exported = find_otel_span(
span_exporter,
&format!("chat {provider_model}"),
&provider_model,
);
assert_eq!(
otel_attribute(&exported, "selected_model"),
Some(&OtelValue::String(provider_model.to_string().into()))
);
let snapshots = flushed_metrics(exporter, provider);
for metric in ["switchyard.decisions", "switchyard.llm_calls"] {
assert_eq!(
u64_counter_value(
&snapshots,
metric,
&[
("algorithm", name),
("selected_model", provider_model.as_str())
]
),
Some(2)
);
for target in ["efficient", "capable"] {
assert_eq!(
u64_counter_value(
&snapshots,
metric,
&[("algorithm", name), ("selected_model", target)]
),
None
);
}
}
assert_eq!(
u64_counter_value(
&snapshots,
"switchyard.requests",
&[("model", provider_model.as_str())]
),
Some(2)
);
}
Ok(())
}