Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(observability): add GenAI semantic attributes (#224)

nachiketb-nvidia9478b46fb735
diff --git a/Cargo.lock b/Cargo.lock
index a4ccd77b0..67e410ec5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2009,6 +2009,7 @@ dependencies = [
"tokio",
"tokio-stream",
"tracing",
+ "tracing-opentelemetry",
"tracing-subscriber",
]
diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs
index ed8c5b7bc..df6d3b91b 100644
--- a/crates/libsy-llm-client/src/client.rs
+++ b/crates/libsy-llm-client/src/client.rs
@@ -178,6 +178,7 @@ impl TranslatingLlmClient {
let streaming = endpoint.allows_streaming()
&& body.get("stream").and_then(Value::as_bool).unwrap_or(false);
let url = endpoint.url(backend);
+ record_gen_ai_request(&url, model, streaming);
let max_retries = u64::from(backend.max_retries());
let max_attempts = max_retries + 1;
@@ -601,6 +602,22 @@ fn duration_millis(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
+fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {
+ let span = tracing::Span::current();
+ span.record("gen_ai.request.model", model);
+ if streaming {
+ span.record("gen_ai.request.stream", true);
+ }
+ if let Ok(url) = reqwest::Url::parse(url) {
+ if let Some(host) = url.host_str() {
+ span.record("server.address", host);
+ }
+ if let Some(port) = url.port_or_known_default() {
+ span.record("server.port", i64::from(port));
+ }
+ }
+}
+
fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
// Reqwest labels truncated or otherwise unreadable response bodies as decode
// errors, so distinguish them from serde JSON failures at the call site.
diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.toml
index 350c6031b..dc2be5249 100644
--- a/crates/libsy/Cargo.toml
+++ b/crates/libsy/Cargo.toml
@@ -26,10 +26,11 @@ thiserror.workspace = true
tokio.workspace = true
tokio-stream = "0.1"
tracing.workspace = true
+tracing-opentelemetry = "0.33"
[dev-dependencies]
# SDK + in-memory exporter to assert what the observability layer records.
-opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing"] }
+opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
switchyard-llm-client.workspace = true
tokio.workspace = true
tracing-subscriber = "0.3"
diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs
index 84eb41d55..9339d482b 100644
--- a/crates/libsy/src/core/algorithm.rs
+++ b/crates/libsy/src/core/algorithm.rs
@@ -618,7 +618,7 @@ pub trait Algorithm: Send + Sync + 'static {
// One `libsy.run` span covers the whole algorithm task; the driver's
// `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s
// contextual parenting.
- let span = observability::run_span(self.name(), request.metadata.as_ref());
+ let span = observability::run_span(self.name(), &request);
let observed_driver = task_driver.clone();
let handle = tokio::spawn(
async move {
@@ -684,12 +684,51 @@ pub trait Algorithm: Send + Sync + 'static {
skip_all,
fields(
algorithm = observability::algorithm_label(&call.get_routed().ctx),
+ switchyard.algorithm = observability::algorithm_label(&call.get_routed().ctx),
+ switchyard.routing.tier = tracing::field::Empty,
selected_model = call.get_decision().selected_model(),
+ otel.kind = "client",
+ otel.name = %format_args!("chat {}", call.get_decision().selected_model()),
+ gen_ai.operation.name = "chat",
+ gen_ai.request.model = call.get_decision().selected_model(),
+ gen_ai.request.stream = tracing::field::Empty,
+ gen_ai.request.temperature = tracing::field::Empty,
+ gen_ai.request.top_p = tracing::field::Empty,
+ gen_ai.request.top_k = tracing::field::Empty,
+ gen_ai.request.max_tokens = tracing::field::Empty,
+ gen_ai.request.reasoning.level = tracing::field::Empty,
+ gen_ai.output.type = tracing::field::Empty,
+ gen_ai.conversation.id = tracing::field::Empty,
+ server.address = tracing::field::Empty,
+ server.port = tracing::field::Empty,
+ gen_ai.response.id = tracing::field::Empty,
+ gen_ai.response.model = tracing::field::Empty,
+ gen_ai.usage.input_tokens = tracing::field::Empty,
+ gen_ai.usage.output_tokens = tracing::field::Empty,
+ gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
+ gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
+ gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
outcome = tracing::field::Empty,
+ otel.status_code = tracing::field::Empty,
+ error.type = tracing::field::Empty,
error = tracing::field::Empty,
)
)]
async fn serve(call: CallLlmRequest) -> Result<()> {
+ let span = tracing::Span::current();
+ observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
+ if let Some(tier) = call.get_decision().routing_tier() {
+ span.record("switchyard.routing.tier", tier);
+ }
+ if let Some(session_id) = call
+ .get_routed()
+ .request
+ .metadata
+ .as_ref()
+ .and_then(|metadata| metadata.session_id.as_deref())
+ {
+ span.record("gen_ai.conversation.id", session_id);
+ }
let routed = call.get_routed().clone();
let target = routed.decision.selected_model().to_string();
let client =
@@ -703,7 +742,7 @@ pub trait Algorithm: Send + Sync + 'static {
.call(routed.ctx, routed.request, routed.decision)
.await
.map_err(|source| LibsyError::client_call(target, source));
- observability::record_client_call(&result);
+ let result = observability::observe_client_call(result);
call.respond(result)
}
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index fd5ef5839..96fed3d3f 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -30,16 +30,25 @@
//! provider installed at any point in the process lifetime; the cost is
//! negligible next to a model call.
+use std::borrow::Cow;
use std::future::Future;
+use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::OnceLock;
+use std::task::{Context as TaskContext, Poll};
use std::time::{Duration, Instant};
+use futures::Stream;
use opentelemetry::metrics::{Meter, ObservableGauge};
-use opentelemetry::{global, KeyValue};
+use opentelemetry::{global, Array as OtelArray, KeyValue, StringValue, Value as OtelValue};
+use switchyard_protocol::StopReason;
use tracing::Span;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
-use crate::{Context, Decision, Driver, Metadata, Response, Result};
+use crate::{
+ AggLlmResponse, Context, Decision, Driver, LibsyError, LlmClientError, LlmRequest, LlmResponse,
+ LlmResponseChunk, LlmResponseStream, Request, Response, Result, Usage,
+};
const METRICS_SCOPE: &str = "switchyard";
const TRACING_TARGET: &str = "libsy";
@@ -96,17 +105,20 @@ fn outcome_value<T>(result: &Result<T>) -> &'static str {
/// Span covering one algorithm run (the whole `create_run_task` execution).
///
-/// Correlation ids from the request [`Metadata`] are recorded as span fields
+/// Correlation ids from the request [`crate::Metadata`] are recorded as span fields
/// when present. `tracing` spans cannot grow field names at runtime, so
-/// arbitrary host labels ride in via [`Metadata::extra_metadata`], recorded
+/// arbitrary host labels ride in via [`crate::Metadata::extra_metadata`], recorded
/// whole into the `extra_metadata` field. `outcome` and `error` are filled in
/// by [`record_run`] when the run ends.
-pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {
+pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {
let span = tracing::info_span!(
target: TRACING_TARGET,
"libsy.run",
algorithm,
+ switchyard.algorithm = algorithm,
+ switchyard.route = tracing::field::Empty,
session_id = tracing::field::Empty,
+ session.id = tracing::field::Empty,
agent_id = tracing::field::Empty,
task_id = tracing::field::Empty,
correlation_id = tracing::field::Empty,
@@ -114,7 +126,10 @@ pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {
outcome = tracing::field::Empty,
error = tracing::field::Empty,
);
- if let Some(metadata) = metadata {
+ if let Some(route) = request.requested_model() {
+ span.record("switchyard.route", route);
+ }
+ if let Some(metadata) = &request.metadata {
for (field, value) in [
("session_id", &metadata.session_id),
("agent_id", &metadata.agent_id),
@@ -125,6 +140,9 @@ pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {
span.record(field, value.as_str());
}
}
+ if let Some(session_id) = &metadata.session_id {
+ span.record("session.id", session_id.as_str());
+ }
if let Some(extra) = &metadata.extra_metadata {
span.record("extra_metadata", tracing::field::debug(extra));
}
@@ -156,17 +174,279 @@ pub(crate) async fn observe_run(
result
}
-/// Records the outcome fields on the enclosing `libsy.client_call` span. The
-/// failure itself is not logged here — it propagates to the algorithm, where
-/// the `libsy.llm_call` recording logs it once.
-pub(crate) fn record_client_call(result: &Result<Response>) {
+/// Records request parameters represented directly by the neutral IR.
+pub(crate) fn record_gen_ai_request(span: &Span, request: &LlmRequest) {
+ if request.stream {
+ span.record("gen_ai.request.stream", true);
+ }
+ if let Some(value) = request.sampling.temperature {
+ span.record("gen_ai.request.temperature", value);
+ }
+ if let Some(value) = request.sampling.top_p {
+ span.record("gen_ai.request.top_p", value);
+ }
+ if let Some(value) = request.sampling.top_k {
+ span.record("gen_ai.request.top_k", value);
+ }
+ if let Some(value) = request.output.max_output_tokens {
+ span.record("gen_ai.request.max_tokens", otel_int(value));
+ }
+ if let Some(value) = request.reasoning.effort.as_deref() {
+ span.record("gen_ai.request.reasoning.level", value);
+ }
+ if let Some(value) = request
+ .output
+ .response_format
+ .as_ref()
+ .and_then(gen_ai_output_type)
+ {
+ span.record("gen_ai.output.type", value);
+ }
+}
+
+/// Adds terminal response and usage fields to the enclosing `libsy.client_call`
+/// span without consuming or buffering a streaming response.
+pub(crate) fn observe_client_call(result: Result<Response>) -> Result<Response> {
let span = Span::current();
- span.record("outcome", outcome_value(result));
- if let Err(error) = result {
- span.record("error", tracing::field::display(error));
+ match result {
+ Ok(mut response) => {
+ match response.llm_response {
+ LlmResponse::Agg(agg) => {
+ span.record("outcome", "ok");
+ record_gen_ai_response(&span, &agg);
+ response.llm_response = LlmResponse::Agg(agg);
+ }
+ LlmResponse::Stream(stream) => {
+ response.llm_response =
+ LlmResponse::Stream(observe_client_stream(stream, span));
+ }
+ }
+ Ok(response)
+ }
+ Err(error) => {
+ let error_type = client_call_error_type(&error);
+ record_client_error(&span, &error_type, &error);
+ Err(error)
+ }
}
}
+fn observe_client_stream(stream: LlmResponseStream, span: Span) -> LlmResponseStream {
+ Box::pin(ObservedClientStream {
+ stream,
+ observer: Some(ClientStreamObserver {
+ span,
+ terminal: false,
+ }),
+ })
+}
+
+fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {
+ record_optional(span, "gen_ai.response.id", response.id.as_deref());
+ record_optional(span, "gen_ai.response.model", response.model.as_deref());
+ record_finish_reasons(
+ span,
+ response
+ .outputs
+ .iter()
+ .filter_map(|output| output.stop_reason)
+ .map(stop_reason_name),
+ );
+ record_gen_ai_usage(span, &response.usage);
+}
+
+fn record_gen_ai_usage(span: &Span, usage: &Usage) {
+ let cache_read = usage.cached_input_tokens();
+ let cache_creation = usage.cache_creation_input_tokens();
+ if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {
+ let input_tokens = usage
+ .input_tokens
+ .unwrap_or_default()
+ .saturating_add(cache_read.unwrap_or_default())
+ .saturating_add(cache_creation.unwrap_or_default());
+ span.record("gen_ai.usage.input_tokens", otel_int(input_tokens));
+ }
+ for (field, value) in [
+ ("gen_ai.usage.output_tokens", usage.output_tokens),
+ ("gen_ai.usage.cache_read.input_tokens", cache_read),
+ ("gen_ai.usage.cache_creation.input_tokens", cache_creation),
+ (
+ "gen_ai.usage.reasoning.output_tokens",
+ usage.reasoning_tokens,
+ ),
+ ] {
+ if let Some(value) = value {
+ span.record(field, otel_int(value));
+ }
+ }
+}
+
+// OpenTelemetry integer attributes are signed; token counts are unsigned in the IR.
+fn otel_int(value: u64) -> i64 {
+ value.min(i64::MAX as u64) as i64
+}
+
+fn record_optional(span: &Span, field: &str, value: Option<&str>) {
+ if let Some(value) = value {
+ span.record(field, value);
+ }
+}
+
+// `tracing` fields cannot preserve a typed string array, so write this attribute directly.
+fn record_finish_reasons(span: &Span, reasons: impl IntoIterator<Item = impl Into<String>>) {
+ let reasons = reasons
+ .into_iter()
+ .map(|reason| StringValue::from(reason.into()))
+ .collect::<Vec<_>>();
+ if !reasons.is_empty() {
+ span.set_attribute(
+ "gen_ai.response.finish_reasons",
+ OtelValue::Array(OtelArray::String(reasons)),
+ );
+ }
+}
+
+fn stop_reason_name(reason: StopReason) -> &'static str {
+ match reason {
+ StopReason::EndTurn => "end_turn",
+ StopReason::MaxTokens => "max_tokens",
+ StopReason::ToolUse => "tool_use",
+ StopReason::ContentFilter => "content_filter",
+ StopReason::Error => "error",
+ StopReason::Unknown => "unknown",
+ }
+}
+
+fn gen_ai_output_type(response_format: &serde_json::Value) -> Option<&'static str> {
+ match response_format
+ .get("type")
+ .and_then(serde_json::Value::as_str)
+ {
+ Some("json" | "json_object" | "json_schema") => Some("json"),
+ Some("text") => Some("text"),
+ _ => None,
+ }
+}
+
+fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {
+ match error {
+ LibsyError::ClientCall { source, .. } => llm_client_error_type(source),
+ LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),
+ LibsyError::NoTargets => Cow::Borrowed("no_targets"),
+ LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),
+ LibsyError::MissingClient { .. } => Cow::Borrowed("client_configuration_error"),
+ LibsyError::Driver(_) => Cow::Borrowed("driver_error"),
+ LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),
+ LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),
+ LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),
+ LibsyError::External { .. } => Cow::Borrowed("_OTHER"),
+ }
+}
+
+fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {
+ match error {
+ LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"),
+ LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"),
+ LlmClientError::RequestEncoding(_) => Cow::Borrowed("request_encoding"),
+ LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),
+ LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),
+ LlmClientError::Transport { .. } => Cow::Borrowed("transport"),
+ LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),
+ LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),
+ LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.to_string()),
+ LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"),
+ LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"),
+ _ => Cow::Borrowed("_OTHER"),
+ }
+}
+
+// Keep the client span alive until the response is drained, errors, or is abandoned.
+struct ObservedClientStream {
+ stream: LlmResponseStream,
+ observer: Option<ClientStreamObserver>,
+}
+
+impl Stream for ObservedClientStream {
+ type Item = std::result::Result<LlmResponseChunk, LlmClientError>;
+
+ fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {
+ match self.stream.as_mut().poll_next(cx) {
+ Poll::Ready(Some(item)) => {
+ let failed = self
+ .observer
+ .as_mut()
+ .is_some_and(|observer| observer.observe(&item));
+ if failed {
+ self.observer.take();
+ }
+ Poll::Ready(Some(item))
+ }
+ Poll::Ready(None) => {
+ if let Some(mut observer) = self.observer.take() {
+ observer.complete();
+ }
+ Poll::Ready(None)
+ }
+ Poll::Pending => Poll::Pending,
+ }
+ }
+}
+
+struct ClientStreamObserver {
+ span: Span,
+ terminal: bool,
+}
+
+impl ClientStreamObserver {
+ fn observe(&mut self, item: &std::result::Result<LlmResponseChunk, LlmClientError>) -> bool {
+ match item {
+ Ok(LlmResponseChunk::MessageStart { id, model }) => {
+ record_optional(&self.span, "gen_ai.response.id", id.as_deref());
+ record_optional(&self.span, "gen_ai.response.model", model.as_deref());
+ }
+ Ok(LlmResponseChunk::Usage(usage)) => record_gen_ai_usage(&self.span, usage),
+ Ok(LlmResponseChunk::MessageStop { reason }) => {
+ record_finish_reasons(&self.span, reason.iter().cloned());
+ }
+ Ok(LlmResponseChunk::DecodeError { message }) => {
+ record_client_error(&self.span, "response_translation", message);
+ self.terminal = true;
+ }
+ Ok(LlmResponseChunk::StreamError { message }) => {
+ record_client_error(&self.span, "502", message);
+ self.terminal = true;
+ }
+ Err(error) => {
+ let error_type = llm_client_error_type(error);
+ record_client_error(&self.span, &error_type, error);
+ self.terminal = true;
+ }
+ _ => {}
+ }
+ self.terminal
+ }
+
+ fn complete(&mut self) {
+ self.span.record("outcome", "ok");
+ self.terminal = true;
+ }
+}
+
+impl Drop for ClientStreamObserver {
+ fn drop(&mut self) {
+ if !self.terminal {
+ self.span.record("outcome", "cancelled");
+ }
+ }
+}
+
+fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) {
+ span.record("outcome", "error");
+ span.record("otel.status_code", "ERROR");
+ span.record("error.type", error_type);
+ span.record("error", tracing::field::display(error));
+}
+
/// Records the end of one algorithm run: the run counter and duration
/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the
/// run failed.
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index 896929a48..a7a218104 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -17,22 +17,27 @@ use std::time::Duration;
use async_trait::async_trait;
use futures::StreamExt;
+use opentelemetry::trace::TracerProvider as _;
+use opentelemetry::{Array as OtelArray, Value as OtelValue};
use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ResourceMetrics};
use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};
+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};
use parking_lot::Mutex;
+use serde_json::json;
use tracing::field::{Field, Visit};
use tracing::span::{Attributes, Id, Record};
use tracing::{Event, Subscriber};
+use tracing_opentelemetry::OpenTelemetryLayer;
use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt};
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::Layer;
use switchyard_libsy::algorithms::{LlmTaskClassifier, TaskClassifierConfig};
use switchyard_libsy::{
- AggLlmResponse, Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmTarget,
+ Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmResponseChunk, LlmTarget,
LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, Step, Usage,
};
-use switchyard_protocol::{text_request, text_response, LlmClientError};
+use switchyard_protocol::{text_request, text_response, LlmClientError, StopReason};
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
@@ -92,6 +97,14 @@ impl Visit for FieldVisitor<'_> {
fn record_u64(&mut self, field: &Field, value: u64) {
self.0.insert(field.name().to_string(), value.to_string());
}
+
+ fn record_i64(&mut self, field: &Field, value: i64) {
+ self.0.insert(field.name().to_string(), value.to_string());
+ }
+
+ fn record_f64(&mut self, field: &Field, value: f64) {
+ self.0.insert(field.name().to_string(), value.to_string());
+ }
}
/// Subscriber layer capturing spans (with contextual parents and recorded
@@ -146,9 +159,16 @@ where
/// Installs the process-global telemetry sinks once: an in-memory OTel metric
/// pipeline behind the global meter provider, and the capture layer as the
/// global tracing subscriber.
-fn telemetry() -> &'static (CaptureStore, InMemoryMetricExporter, SdkMeterProvider) {
- static TELEMETRY: OnceLock<(CaptureStore, InMemoryMetricExporter, SdkMeterProvider)> =
- OnceLock::new();
+type Telemetry = (
+ CaptureStore,
+ InMemoryMetricExporter,
+ SdkMeterProvider,
+ InMemorySpanExporter,
+ SdkTracerProvider,
+);
+
+fn telemetry() -> &'static Telemetry {
+ static TELEMETRY: OnceLock<Telemetry> = OnceLock::new();
TELEMETRY.get_or_init(|| {
let exporter = InMemoryMetricExporter::default();
let reader = PeriodicReader::builder(exporter.clone()).build();
@@ -156,18 +176,27 @@ fn telemetry() -> &'static (CaptureStore, InMemoryMetricExporter, SdkMeterProvid
opentelemetry::global::set_meter_provider(provider.clone());
switchyard_libsy::initialize_metrics();
+ let span_exporter = InMemorySpanExporter::default();
+ let tracer_provider = SdkTracerProvider::builder()
+ .with_simple_exporter(span_exporter.clone())
+ .build();
+ let tracer = tracer_provider.tracer("switchyard-observability-test");
let store = CaptureStore::default();
- let subscriber = tracing_subscriber::registry().with(CaptureLayer {
- store: store.clone(),
- });
+ let otel_layer: OpenTelemetryLayer<_, _> =
+ tracing_opentelemetry::layer().with_tracer(tracer);
+ let subscriber = tracing_subscriber::registry()
+ .with(CaptureLayer {
+ store: store.clone(),
+ })
+ .with(otel_layer);
if tracing::subscriber::set_global_default(subscriber).is_err() {
panic!("a global tracing subscriber was already installed in this test binary");
}
- (store, exporter, provider)
+ (store, exporter, provider, span_exporter, tracer_provider)
})
}
-/// The three tests in this file must not overlap because metrics are global.
+/// The tests in this file must not overlap because metrics are global.
/// There is no Rust/cargo-native way of saying this (people use `serial_test` crate) so use a
/// lock.
/// Each file in `tests/` (integration tests) runs as a separate test process, so we are not
@@ -343,12 +372,15 @@ impl RoutedLlmClient for UsageClient {
_request: Request,
decision: Arc<dyn Decision>,
) -> Result<Response, switchyard_protocol::LlmClientError> {
+ let mut response = text_response(
+ Some(decision.selected_model().to_string()),
+ "observed response",
+ );
+ response.id = Some("obs-response-1".to_string());
+ response.usage = self.usage.clone();
+ response.outputs[0].stop_reason = Some(StopReason::EndTurn);
Ok(Response {
- llm_response: LlmResponse::Agg(AggLlmResponse {
- model: Some(decision.selected_model().to_string()),
- usage: self.usage.clone(),
- ..AggLlmResponse::default()
- }),
+ llm_response: LlmResponse::Agg(response),
metadata: None,
})
}
@@ -426,10 +458,47 @@ fn find_span(spans: &[SpanRecord], name: &str, field: &str, value: &str) -> Span
}
}
+fn find_otel_span(exporter: &InMemorySpanExporter, name: &str, model: &str) -> SpanData {
+ let spans = match exporter.get_finished_spans() {
+ Ok(spans) => spans,
+ Err(error) => panic!("failed to read exported spans: {error}"),
+ };
+ match spans.iter().find(|span| {
+ span.name == name
+ && span.attributes.iter().any(|attribute| {
+ attribute.key.as_str() == "gen_ai.request.model"
+ && attribute.value.as_str() == model
+ })
+ }) {
+ Some(span) => span.clone(),
+ None => {
+ let available = spans
+ .iter()
+ .map(|span| {
+ let model = span
+ .attributes
+ .iter()
+ .find(|attribute| attribute.key.as_str() == "gen_ai.request.model")
+ .map(|attribute| attribute.value.as_str().into_owned());
+ (span.name.to_string(), model)
+ })
+ .collect::<Vec<_>>();
+ panic!("no exported '{name}' span for model {model}; available: {available:?}")
+ }
+ }
+}
+
+fn otel_attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a OtelValue> {
+ span.attributes
+ .iter()
+ .find(|attribute| attribute.key.as_str() == key)
+ .map(|attribute| &attribute.value)
+}
+
#[tokio::test]
async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
- let (store, exporter, provider) = telemetry();
+ let (store, exporter, provider, span_exporter, _) = telemetry();
const ALGO: &str = "obs-success-algo";
const MODEL: &str = "obs-success-model";
let before = flushed_metrics(exporter, provider);
@@ -442,16 +511,20 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
usage: Usage {
input_tokens: Some(11),
output_tokens: Some(7),
- total_tokens: Some(18),
+ total_tokens: Some(25),
reasoning_tokens: Some(2),
- ..Usage::default()
+ cache: Usage::cache_details(Some(3), Some(4)),
},
}) as Arc<dyn RoutedLlmClient>;
+ let mut request = request_with_metadata("obs-session-1", "obs-corr-1");
+ request.llm_request.sampling.temperature = Some(0.25);
+ request.llm_request.sampling.top_p = Some(0.9);
+ request.llm_request.sampling.top_k = Some(40);
+ request.llm_request.output.max_output_tokens = Some(512);
+ request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));
+ request.llm_request.reasoning.effort = Some("high".to_string());
let (trace, _response) = algo(ALGO, MODEL, Some(client))
- .run(
- Context::default(),
- request_with_metadata("obs-session-1", "obs-corr-1"),
- )
+ .run(Context::default(), request)
.await?;
assert_eq!(trace.len(), 1);
@@ -525,6 +598,21 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
run_span.fields.get("session_id").map(String::as_str),
Some("obs-session-1")
);
+ assert_eq!(
+ run_span.fields.get("session.id").map(String::as_str),
+ Some("obs-session-1")
+ );
+ assert_eq!(
+ run_span
+ .fields
+ .get("switchyard.algorithm")
+ .map(String::as_str),
+ Some(ALGO)
+ );
+ assert_eq!(
+ run_span.fields.get("switchyard.route").map(String::as_str),
+ Some("auto")
+ );
assert_eq!(
run_span.fields.get("correlation_id").map(String::as_str),
Some("obs-corr-1")
@@ -555,6 +643,49 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
client_span.fields.get("outcome").map(String::as_str),
Some("ok")
);
+ for (field, value) in [
+ ("otel.kind", "client"),
+ ("switchyard.algorithm", ALGO),
+ ("otel.name", "chat obs-success-model"),
+ ("gen_ai.operation.name", "chat"),
+ ("gen_ai.request.model", MODEL),
+ ("gen_ai.request.temperature", "0.25"),
+ ("gen_ai.request.top_p", "0.9"),
+ ("gen_ai.request.top_k", "40"),
+ ("gen_ai.request.max_tokens", "512"),
+ ("gen_ai.request.reasoning.level", "high"),
+ ("gen_ai.output.type", "json"),
+ ("gen_ai.conversation.id", "obs-session-1"),
+ ("gen_ai.response.id", "obs-response-1"),
+ ("gen_ai.response.model", MODEL),
+ ("gen_ai.usage.input_tokens", "18"),
+ ("gen_ai.usage.output_tokens", "7"),
+ ("gen_ai.usage.cache_read.input_tokens", "3"),
+ ("gen_ai.usage.cache_creation.input_tokens", "4"),
+ ("gen_ai.usage.reasoning.output_tokens", "2"),
+ ] {
+ assert_eq!(
+ client_span.fields.get(field).map(String::as_str),
+ Some(value),
+ "unexpected {field}"
+ );
+ }
+ assert_eq!(client_span.fields.get("gen_ai.request.stream"), None);
+
+ let otel_span = find_otel_span(span_exporter, "chat obs-success-model", MODEL);
+ assert!(matches!(
+ otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),
+ Some(OtelValue::Array(OtelArray::String(reasons)))
+ if reasons.len() == 1 && reasons[0].as_str() == "end_turn"
+ ));
+ assert_eq!(
+ otel_attribute(&otel_span, "gen_ai.request.max_tokens"),
+ Some(&OtelValue::I64(512))
+ );
+ assert_eq!(
+ otel_attribute(&otel_span, "gen_ai.usage.input_tokens"),
+ Some(&OtelValue::I64(18))
+ );
let call_span = find_span(&spans, "libsy.llm_call", "selected_model", MODEL);
assert_eq!(call_span.parent.as_deref(), Some("libsy.run"));
@@ -576,7 +707,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
);
assert_eq!(
call_span.fields.get("total_tokens").map(String::as_str),
- Some("18")
+ Some("25")
);
assert_eq!(
call_span.fields.get("reasoning_tokens").map(String::as_str),
@@ -604,10 +735,165 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
Ok(())
}
+/// A streamed response keeps the client span available until terminal usage arrives.
+struct StreamingUsageClient;
+
+#[async_trait]
+impl RoutedLlmClient for StreamingUsageClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ decision: Arc<dyn Decision>,
+ ) -> Result<Response, LlmClientError> {
+ let usage = Usage {
+ input_tokens: Some(13),
+ output_tokens: Some(5),
+ cache: Usage::cache_details(Some(8), None),
+ ..Usage::default()
+ };
+ let chunks = vec![
+ Ok(LlmResponseChunk::MessageStart {
+ id: Some("obs-stream-response".to_string()),
+ model: Some(decision.selected_model().to_string()),
+ }),
+ Ok(LlmResponseChunk::Usage(usage)),
+ Ok(LlmResponseChunk::MessageStop {
+ reason: Some("end_turn".to_string()),
+ }),
+ ];
+ Ok(Response {
+ llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))),
+ metadata: None,
+ })
+ }
+}
+
+struct TimeoutClient;
+
+#[async_trait]
+impl RoutedLlmClient for TimeoutClient {
+ async fn call(
+ &self,
+ _ctx: Context,
+ _request: Request,
+ _decision: Arc<dyn Decision>,
+ ) -> Result<Response, LlmClientError> {
+ Err(LlmClientError::Timeout {
+ source: Box::new(TestError("upstream timed out")),
+ })
+ }
+}
+
+#[tokio::test]
+async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
+ let (store, _, _, span_exporter, _) = telemetry();
+ const ALGO: &str = "obs-stream-algo";
+ const MODEL: &str = "obs-stream-model";
+ let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;
+ let mut request = request_with_metadata("obs-stream-session", "obs-stream-corr");
+ request.llm_request.stream = true;
+ let (_, response) = algo(ALGO, MODEL, Some(client))
+ .run(Context::default(), request)
+ .await?;
+ let LlmResponse::Stream(mut stream) = response.llm_response else {
+ return Err(test_error("expected a streamed response"));
+ };
+ while let Some(item) = stream.next().await {
+ if let Err(error) = item {
+ panic!("unexpected stream error: {error}");
+ }
+ }
+
+ let spans = store.spans();
+ let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
+ for (field, value) in [
+ ("otel.name", "chat obs-stream-model"),
+ ("gen_ai.request.stream", "true"),
+ ("gen_ai.response.id", "obs-stream-response"),
+ ("gen_ai.response.model", MODEL),
+ ("gen_ai.usage.input_tokens", "21"),
+ ("gen_ai.usage.output_tokens", "5"),
+ ("gen_ai.usage.cache_read.input_tokens", "8"),
+ ] {
+ assert_eq!(
+ client_span.fields.get(field).map(String::as_str),
+ Some(value),
+ "unexpected {field}"
+ );
+ }
+ let otel_span = find_otel_span(span_exporter, "chat obs-stream-model", MODEL);
+ assert!(matches!(
+ otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),
+ Some(OtelValue::Array(OtelArray::String(reasons)))
+ if reasons.len() == 1 && reasons[0].as_str() == "end_turn"
+ ));
+ Ok(())
+}
+
+#[tokio::test]
+async fn dropped_stream_records_cancelled_outcome() -> switchyard_libsy::Result<()> {
+ let _guard = serialize_test().lock().await;
+ let (store, _, _, _, _) = telemetry();
+ const ALGO: &str = "obs-cancelled-stream-algo";
+ const MODEL: &str = "obs-cancelled-stream-model";
+ let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;
+ let mut request = request_with_metadata("obs-cancelled-session", "obs-cancelled-corr");
+ request.llm_request.stream = true;
+ let (_, response) = algo(ALGO, MODEL, Some(client))
+ .run(Context::default(), request)
+ .await?;
+ let LlmResponse::Stream(stream) = response.llm_response else {
+ return Err(test_error("expected a streamed response"));
+ };
+ drop(stream);
+
+ let spans = store.spans();
+ let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
+ assert_eq!(
+ client_span.fields.get("outcome").map(String::as_str),
+ Some("cancelled")
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn typed_client_failure_records_semantic_error_type() {
+ let _guard = serialize_test().lock().await;
+ let (store, _, _, _, _) = telemetry();
+ const ALGO: &str = "obs-timeout-algo";
+ const MODEL: &str = "obs-timeout-model";
+ let result = algo(
+ ALGO,
+ MODEL,
+ Some(Arc::new(TimeoutClient) as Arc<dyn RoutedLlmClient>),
+ )
+ .run(
+ Context::default(),
+ request_with_metadata("obs-timeout-session", "obs-timeout-corr"),
+ )
+ .await;
+ assert!(matches!(
+ result,
+ Err(LibsyError::ClientCall {
+ source: LlmClientError::Timeout { .. },
+ ..
+ })
+ ));
+
+ let spans = store.spans();
+ let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);
+ assert_eq!(
+ client_span.fields.get("error.type").map(String::as_str),
+ Some("timeout")
+ );
+}
+
#[tokio::test]
async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
- let (store, exporter, provider) = telemetry();
+ let (store, exporter, provider, _, _) = telemetry();
const ALGO: &str = "obs-failure-algo";
const MODEL: &str = "obs-failure-model";
let before = flushed_metrics(exporter, provider);
@@ -729,7 +1015,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
#[tokio::test]
async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_libsy::Result<()> {
let _guard = serialize_test().lock().await;
- let (_store, exporter, provider) = telemetry();
+ let (_store, exporter, provider, _, _) = telemetry();
let before = flushed_metrics(exporter, provider);
let total_requests_before =
u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 9e3ee3028..be00ffc59 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -35,7 +35,7 @@ use libsy::{
};
use serde_json::{json, Value};
use tokio::net::{TcpListener, TcpSocket};
-use tracing::Level;
+use tracing::{Instrument, Level};
use switchyard_translation::{decode_request, WireFormat};
@@ -418,6 +418,19 @@ async fn handle_endpoint(
headers: HeaderMap,
body: std::result::Result<Json<Value>, JsonRejection>,
wire_format: WireFormat,
+) -> Response {
+ let span = observability::request_span(&headers);
+ handle_endpoint_inner(state, started, headers, body, wire_format)
+ .instrument(span)
+ .await
+}
+
+async fn handle_endpoint_inner(
+ state: ServerState,
+ started: RequestStart,
+ headers: HeaderMap,
+ body: std::result::Result<Json<Value>, JsonRejection>,
+ wire_format: WireFormat,
) -> Response {
let metadata = metadata_from_headers(&headers);
let request_log = RequestLogContext {
diff --git a/crates/switchyard-server/src/observability.rs b/crates/switchyard-server/src/observability.rs
index 6f9735bbf..fe3036cf2 100644
--- a/crates/switchyard-server/src/observability.rs
+++ b/crates/switchyard-server/src/observability.rs
@@ -6,9 +6,13 @@
use std::env;
use std::sync::OnceLock;
+use axum::http::HeaderMap;
+use opentelemetry::propagation::{Extractor, TextMapPropagator};
use opentelemetry::trace::TracerProvider as _;
+use opentelemetry_sdk::propagation::TraceContextPropagator;
use opentelemetry_sdk::trace::SdkTracerProvider;
use opentelemetry_sdk::Resource;
+use tracing_opentelemetry::OpenTelemetrySpanExt;
use tracing_subscriber::layer::SubscriberExt as _;
use tracing_subscriber::util::SubscriberInitExt as _;
use tracing_subscriber::{EnvFilter, Layer as _};
@@ -44,6 +48,30 @@ pub fn flush_observability() {
metrics::flush();
}
+/// Creates the server request span with any incoming W3C trace context as its parent.
+pub(crate) fn request_span(headers: &HeaderMap) -> tracing::Span {
+ let parent = TraceContextPropagator::new().extract(&HeaderExtractor(headers));
+ let span = tracing::info_span!(
+ target: "switchyard_server",
+ "switchyard.request",
+ otel.kind = "server",
+ );
+ let _ = span.set_parent(parent);
+ span
+}
+
+struct HeaderExtractor<'a>(&'a HeaderMap);
+
+impl Extractor for HeaderExtractor<'_> {
+ fn get(&self, key: &str) -> Option<&str> {
+ self.0.get(key).and_then(|value| value.to_str().ok())
+ }
+
+ fn keys(&self) -> Vec<&str> {
+ self.0.keys().map(|name| name.as_str()).collect()
+ }
+}
+
pub(crate) fn otlp_enabled(signal: &str) -> bool {
if env_var_is_true("OTEL_SDK_DISABLED") {
return false;
@@ -130,3 +158,43 @@ fn build_tracer_provider() -> Result<SdkTracerProvider, String> {
fn env_var_is_true(name: &str) -> bool {
env::var(name).is_ok_and(|value| matches!(value.to_ascii_lowercase().as_str(), "true" | "1"))
}
+
+#[cfg(test)]
+mod tests {
+ use axum::http::{HeaderMap, HeaderValue};
+ use opentelemetry::trace::{TraceContextExt, TracerProvider as _};
+ use opentelemetry_sdk::trace::SdkTracerProvider;
+ use tracing_opentelemetry::OpenTelemetrySpanExt as _;
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ use super::request_span;
+
+ #[test]
+ fn request_span_continues_incoming_w3c_trace_context() {
+ let provider = SdkTracerProvider::builder().build();
+ let tracer = provider.tracer("request-span-test");
+ let subscriber =
+ tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));
+ let mut headers = HeaderMap::new();
+ headers.insert(
+ "traceparent",
+ HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
+ );
+ headers.insert(
+ "tracestate",
+ HeaderValue::from_static("vendor=opaque-value"),
+ );
+
+ tracing::subscriber::with_default(subscriber, || {
+ let span = request_span(&headers);
+ let context = span.context();
+ let current = context.span();
+ let span_context = current.span_context();
+ assert_eq!(
+ span_context.trace_id().to_string(),
+ "4bf92f3577b34da6a3ce929d0e0e4736"
+ );
+ assert_eq!(span_context.trace_state().header(), "vendor=opaque-value");
+ });
+ }
+}