Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.
Rust
feat(observability): add GenAI semantic attributes (#224)
nachiketb-nvidia9478b46fb735
diff --git a/Cargo.lock b/Cargo.lockindex a4ccd77b0..67e410ec5 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2009,6 +2009,7 @@ dependencies = ["tokio","tokio-stream","tracing",+ "tracing-opentelemetry","tracing-subscriber",]diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rsindex ed8c5b7bc..df6d3b91b 100644--- a/crates/libsy-llm-client/src/client.rs+++ b/crates/libsy-llm-client/src/client.rs@@ -178,6 +178,7 @@ impl TranslatingLlmClient {let streaming = endpoint.allows_streaming()&& body.get("stream").and_then(Value::as_bool).unwrap_or(false);let url = endpoint.url(backend);+ record_gen_ai_request(&url, model, streaming);let max_retries = u64::from(backend.max_retries());let max_attempts = max_retries + 1;@@ -601,6 +602,22 @@ fn duration_millis(duration: Duration) -> u64 {u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)}+fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {+ let span = tracing::Span::current();+ span.record("gen_ai.request.model", model);+ if streaming {+ span.record("gen_ai.request.stream", true);+ }+ if let Ok(url) = reqwest::Url::parse(url) {+ if let Some(host) = url.host_str() {+ span.record("server.address", host);+ }+ if let Some(port) = url.port_or_known_default() {+ span.record("server.port", i64::from(port));+ }+ }+}+fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {// Reqwest labels truncated or otherwise unreadable response bodies as decode// errors, so distinguish them from serde JSON failures at the call site.diff --git a/crates/libsy/Cargo.toml b/crates/libsy/Cargo.tomlindex 350c6031b..dc2be5249 100644--- a/crates/libsy/Cargo.toml+++ b/crates/libsy/Cargo.toml@@ -26,10 +26,11 @@ thiserror.workspace = truetokio.workspace = truetokio-stream = "0.1"tracing.workspace = true+tracing-opentelemetry = "0.33"[dev-dependencies]# SDK + in-memory exporter to assert what the observability layer records.-opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing"] }+opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }switchyard-llm-client.workspace = truetokio.workspace = truetracing-subscriber = "0.3"diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rsindex 84eb41d55..9339d482b 100644--- a/crates/libsy/src/core/algorithm.rs+++ b/crates/libsy/src/core/algorithm.rs@@ -618,7 +618,7 @@ pub trait Algorithm: Send + Sync + 'static {// One `libsy.run` span covers the whole algorithm task; the driver's// `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s// contextual parenting.- let span = observability::run_span(self.name(), request.metadata.as_ref());+ let span = observability::run_span(self.name(), &request);let observed_driver = task_driver.clone();let handle = tokio::spawn(async move {@@ -684,12 +684,51 @@ pub trait Algorithm: Send + Sync + 'static {skip_all,fields(algorithm = observability::algorithm_label(&call.get_routed().ctx),+ switchyard.algorithm = observability::algorithm_label(&call.get_routed().ctx),+ switchyard.routing.tier = tracing::field::Empty,selected_model = call.get_decision().selected_model(),+ otel.kind = "client",+ otel.name = %format_args!("chat {}", call.get_decision().selected_model()),+ gen_ai.operation.name = "chat",+ gen_ai.request.model = call.get_decision().selected_model(),+ gen_ai.request.stream = tracing::field::Empty,+ gen_ai.request.temperature = tracing::field::Empty,+ gen_ai.request.top_p = tracing::field::Empty,+ gen_ai.request.top_k = tracing::field::Empty,+ gen_ai.request.max_tokens = tracing::field::Empty,+ gen_ai.request.reasoning.level = tracing::field::Empty,+ gen_ai.output.type = tracing::field::Empty,+ gen_ai.conversation.id = tracing::field::Empty,+ server.address = tracing::field::Empty,+ server.port = tracing::field::Empty,+ gen_ai.response.id = tracing::field::Empty,+ gen_ai.response.model = tracing::field::Empty,+ gen_ai.usage.input_tokens = tracing::field::Empty,+ gen_ai.usage.output_tokens = tracing::field::Empty,+ gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,+ gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,+ gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,outcome = tracing::field::Empty,+ otel.status_code = tracing::field::Empty,+ error.type = tracing::field::Empty,error = tracing::field::Empty,))]async fn serve(call: CallLlmRequest) -> Result<()> {+ let span = tracing::Span::current();+ observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);+ if let Some(tier) = call.get_decision().routing_tier() {+ span.record("switchyard.routing.tier", tier);+ }+ if let Some(session_id) = call+ .get_routed()+ .request+ .metadata+ .as_ref()+ .and_then(|metadata| metadata.session_id.as_deref())+ {+ span.record("gen_ai.conversation.id", session_id);+ }let routed = call.get_routed().clone();let target = routed.decision.selected_model().to_string();let client =@@ -703,7 +742,7 @@ pub trait Algorithm: Send + Sync + 'static {.call(routed.ctx, routed.request, routed.decision).await.map_err(|source| LibsyError::client_call(target, source));- observability::record_client_call(&result);+ let result = observability::observe_client_call(result);call.respond(result)}diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rsindex fd5ef5839..96fed3d3f 100644--- a/crates/libsy/src/observability.rs+++ b/crates/libsy/src/observability.rs@@ -30,16 +30,25 @@//! provider installed at any point in the process lifetime; the cost is//! negligible next to a model call.+use std::borrow::Cow;use std::future::Future;+use std::pin::Pin;use std::sync::atomic::{AtomicU64, Ordering};use std::sync::OnceLock;+use std::task::{Context as TaskContext, Poll};use std::time::{Duration, Instant};+use futures::Stream;use opentelemetry::metrics::{Meter, ObservableGauge};-use opentelemetry::{global, KeyValue};+use opentelemetry::{global, Array as OtelArray, KeyValue, StringValue, Value as OtelValue};+use switchyard_protocol::StopReason;use tracing::Span;+use tracing_opentelemetry::OpenTelemetrySpanExt;-use crate::{Context, Decision, Driver, Metadata, Response, Result};+use crate::{+ AggLlmResponse, Context, Decision, Driver, LibsyError, LlmClientError, LlmRequest, LlmResponse,+ LlmResponseChunk, LlmResponseStream, Request, Response, Result, Usage,+};const METRICS_SCOPE: &str = "switchyard";const TRACING_TARGET: &str = "libsy";@@ -96,17 +105,20 @@ fn outcome_value<T>(result: &Result<T>) -> &'static str {/// Span covering one algorithm run (the whole `create_run_task` execution).///-/// Correlation ids from the request [`Metadata`] are recorded as span fields+/// Correlation ids from the request [`crate::Metadata`] are recorded as span fields/// when present. `tracing` spans cannot grow field names at runtime, so-/// arbitrary host labels ride in via [`Metadata::extra_metadata`], recorded+/// arbitrary host labels ride in via [`crate::Metadata::extra_metadata`], recorded/// whole into the `extra_metadata` field. `outcome` and `error` are filled in/// by [`record_run`] when the run ends.-pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {+pub(crate) fn run_span(algorithm: &str, request: &Request) -> Span {let span = tracing::info_span!(target: TRACING_TARGET,"libsy.run",algorithm,+ switchyard.algorithm = algorithm,+ switchyard.route = tracing::field::Empty,session_id = tracing::field::Empty,+ session.id = tracing::field::Empty,agent_id = tracing::field::Empty,task_id = tracing::field::Empty,correlation_id = tracing::field::Empty,@@ -114,7 +126,10 @@ pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {outcome = tracing::field::Empty,error = tracing::field::Empty,);- if let Some(metadata) = metadata {+ if let Some(route) = request.requested_model() {+ span.record("switchyard.route", route);+ }+ if let Some(metadata) = &request.metadata {for (field, value) in [("session_id", &metadata.session_id),("agent_id", &metadata.agent_id),@@ -125,6 +140,9 @@ pub(crate) fn run_span(algorithm: &str, metadata: Option<&Metadata>) -> Span {span.record(field, value.as_str());}}+ if let Some(session_id) = &metadata.session_id {+ span.record("session.id", session_id.as_str());+ }if let Some(extra) = &metadata.extra_metadata {span.record("extra_metadata", tracing::field::debug(extra));}@@ -156,17 +174,279 @@ pub(crate) async fn observe_run(result}-/// Records the outcome fields on the enclosing `libsy.client_call` span. The-/// failure itself is not logged here — it propagates to the algorithm, where-/// the `libsy.llm_call` recording logs it once.-pub(crate) fn record_client_call(result: &Result<Response>) {+/// Records request parameters represented directly by the neutral IR.+pub(crate) fn record_gen_ai_request(span: &Span, request: &LlmRequest) {+ if request.stream {+ span.record("gen_ai.request.stream", true);+ }+ if let Some(value) = request.sampling.temperature {+ span.record("gen_ai.request.temperature", value);+ }+ if let Some(value) = request.sampling.top_p {+ span.record("gen_ai.request.top_p", value);+ }+ if let Some(value) = request.sampling.top_k {+ span.record("gen_ai.request.top_k", value);+ }+ if let Some(value) = request.output.max_output_tokens {+ span.record("gen_ai.request.max_tokens", otel_int(value));+ }+ if let Some(value) = request.reasoning.effort.as_deref() {+ span.record("gen_ai.request.reasoning.level", value);+ }+ if let Some(value) = request+ .output+ .response_format+ .as_ref()+ .and_then(gen_ai_output_type)+ {+ span.record("gen_ai.output.type", value);+ }+}++/// Adds terminal response and usage fields to the enclosing `libsy.client_call`+/// span without consuming or buffering a streaming response.+pub(crate) fn observe_client_call(result: Result<Response>) -> Result<Response> {let span = Span::current();- span.record("outcome", outcome_value(result));- if let Err(error) = result {- span.record("error", tracing::field::display(error));+ match result {+ Ok(mut response) => {+ match response.llm_response {+ LlmResponse::Agg(agg) => {+ span.record("outcome", "ok");+ record_gen_ai_response(&span, &agg);+ response.llm_response = LlmResponse::Agg(agg);+ }+ LlmResponse::Stream(stream) => {+ response.llm_response =+ LlmResponse::Stream(observe_client_stream(stream, span));+ }+ }+ Ok(response)+ }+ Err(error) => {+ let error_type = client_call_error_type(&error);+ record_client_error(&span, &error_type, &error);+ Err(error)+ }}}+fn observe_client_stream(stream: LlmResponseStream, span: Span) -> LlmResponseStream {+ Box::pin(ObservedClientStream {+ stream,+ observer: Some(ClientStreamObserver {+ span,+ terminal: false,+ }),+ })+}++fn record_gen_ai_response(span: &Span, response: &AggLlmResponse) {+ record_optional(span, "gen_ai.response.id", response.id.as_deref());+ record_optional(span, "gen_ai.response.model", response.model.as_deref());+ record_finish_reasons(+ span,+ response+ .outputs+ .iter()+ .filter_map(|output| output.stop_reason)+ .map(stop_reason_name),+ );+ record_gen_ai_usage(span, &response.usage);+}++fn record_gen_ai_usage(span: &Span, usage: &Usage) {+ let cache_read = usage.cached_input_tokens();+ let cache_creation = usage.cache_creation_input_tokens();+ if usage.input_tokens.is_some() || cache_read.is_some() || cache_creation.is_some() {+ let input_tokens = usage+ .input_tokens+ .unwrap_or_default()+ .saturating_add(cache_read.unwrap_or_default())+ .saturating_add(cache_creation.unwrap_or_default());+ span.record("gen_ai.usage.input_tokens", otel_int(input_tokens));+ }+ for (field, value) in [+ ("gen_ai.usage.output_tokens", usage.output_tokens),+ ("gen_ai.usage.cache_read.input_tokens", cache_read),+ ("gen_ai.usage.cache_creation.input_tokens", cache_creation),+ (+ "gen_ai.usage.reasoning.output_tokens",+ usage.reasoning_tokens,+ ),+ ] {+ if let Some(value) = value {+ span.record(field, otel_int(value));+ }+ }+}++// OpenTelemetry integer attributes are signed; token counts are unsigned in the IR.+fn otel_int(value: u64) -> i64 {+ value.min(i64::MAX as u64) as i64+}++fn record_optional(span: &Span, field: &str, value: Option<&str>) {+ if let Some(value) = value {+ span.record(field, value);+ }+}++// `tracing` fields cannot preserve a typed string array, so write this attribute directly.+fn record_finish_reasons(span: &Span, reasons: impl IntoIterator<Item = impl Into<String>>) {+ let reasons = reasons+ .into_iter()+ .map(|reason| StringValue::from(reason.into()))+ .collect::<Vec<_>>();+ if !reasons.is_empty() {+ span.set_attribute(+ "gen_ai.response.finish_reasons",+ OtelValue::Array(OtelArray::String(reasons)),+ );+ }+}++fn stop_reason_name(reason: StopReason) -> &'static str {+ match reason {+ StopReason::EndTurn => "end_turn",+ StopReason::MaxTokens => "max_tokens",+ StopReason::ToolUse => "tool_use",+ StopReason::ContentFilter => "content_filter",+ StopReason::Error => "error",+ StopReason::Unknown => "unknown",+ }+}++fn gen_ai_output_type(response_format: &serde_json::Value) -> Option<&'static str> {+ match response_format+ .get("type")+ .and_then(serde_json::Value::as_str)+ {+ Some("json" | "json_object" | "json_schema") => Some("json"),+ Some("text") => Some("text"),+ _ => None,+ }+}++fn client_call_error_type(error: &LibsyError) -> Cow<'static, str> {+ match error {+ LibsyError::ClientCall { source, .. } => llm_client_error_type(source),+ LibsyError::TargetNotFound { .. } => Cow::Borrowed("target_not_found"),+ LibsyError::NoTargets => Cow::Borrowed("no_targets"),+ LibsyError::AlgorithmError { .. } => Cow::Borrowed("algorithm_error"),+ LibsyError::MissingClient { .. } => Cow::Borrowed("client_configuration_error"),+ LibsyError::Driver(_) => Cow::Borrowed("driver_error"),+ LibsyError::AlgorithmTask { .. } => Cow::Borrowed("algorithm_task_error"),+ LibsyError::MissingFinalResponse => Cow::Borrowed("missing_final_response"),+ LibsyError::AllTargetsExcluded => Cow::Borrowed("context_window_exceeded"),+ LibsyError::External { .. } => Cow::Borrowed("_OTHER"),+ }+}++fn llm_client_error_type(error: &LlmClientError) -> Cow<'static, str> {+ match error {+ LlmClientError::InvalidRequest { .. } => Cow::Borrowed("invalid_request"),+ LlmClientError::RequestTranslation(_) => Cow::Borrowed("request_translation"),+ LlmClientError::RequestEncoding(_) => Cow::Borrowed("request_encoding"),+ LlmClientError::ResponseTranslation(_) => Cow::Borrowed("response_translation"),+ LlmClientError::Configuration { .. } => Cow::Borrowed("configuration"),+ LlmClientError::Transport { .. } => Cow::Borrowed("transport"),+ LlmClientError::Timeout { .. } => Cow::Borrowed("timeout"),+ LlmClientError::ContextWindowExceeded { .. } => Cow::Borrowed("context_window_exceeded"),+ LlmClientError::UpstreamHttp { status, .. } => Cow::Owned(status.to_string()),+ LlmClientError::InvalidResponse { .. } => Cow::Borrowed("invalid_response"),+ LlmClientError::Ffi { .. } => Cow::Borrowed("ffi"),+ _ => Cow::Borrowed("_OTHER"),+ }+}++// Keep the client span alive until the response is drained, errors, or is abandoned.+struct ObservedClientStream {+ stream: LlmResponseStream,+ observer: Option<ClientStreamObserver>,+}++impl Stream for ObservedClientStream {+ type Item = std::result::Result<LlmResponseChunk, LlmClientError>;++ fn poll_next(mut self: Pin<&mut Self>, cx: &mut TaskContext<'_>) -> Poll<Option<Self::Item>> {+ match self.stream.as_mut().poll_next(cx) {+ Poll::Ready(Some(item)) => {+ let failed = self+ .observer+ .as_mut()+ .is_some_and(|observer| observer.observe(&item));+ if failed {+ self.observer.take();+ }+ Poll::Ready(Some(item))+ }+ Poll::Ready(None) => {+ if let Some(mut observer) = self.observer.take() {+ observer.complete();+ }+ Poll::Ready(None)+ }+ Poll::Pending => Poll::Pending,+ }+ }+}++struct ClientStreamObserver {+ span: Span,+ terminal: bool,+}++impl ClientStreamObserver {+ fn observe(&mut self, item: &std::result::Result<LlmResponseChunk, LlmClientError>) -> bool {+ match item {+ Ok(LlmResponseChunk::MessageStart { id, model }) => {+ record_optional(&self.span, "gen_ai.response.id", id.as_deref());+ record_optional(&self.span, "gen_ai.response.model", model.as_deref());+ }+ Ok(LlmResponseChunk::Usage(usage)) => record_gen_ai_usage(&self.span, usage),+ Ok(LlmResponseChunk::MessageStop { reason }) => {+ record_finish_reasons(&self.span, reason.iter().cloned());+ }+ Ok(LlmResponseChunk::DecodeError { message }) => {+ record_client_error(&self.span, "response_translation", message);+ self.terminal = true;+ }+ Ok(LlmResponseChunk::StreamError { message }) => {+ record_client_error(&self.span, "502", message);+ self.terminal = true;+ }+ Err(error) => {+ let error_type = llm_client_error_type(error);+ record_client_error(&self.span, &error_type, error);+ self.terminal = true;+ }+ _ => {}+ }+ self.terminal+ }++ fn complete(&mut self) {+ self.span.record("outcome", "ok");+ self.terminal = true;+ }+}++impl Drop for ClientStreamObserver {+ fn drop(&mut self) {+ if !self.terminal {+ self.span.record("outcome", "cancelled");+ }+ }+}++fn record_client_error(span: &Span, error_type: &str, error: &dyn std::fmt::Display) {+ span.record("outcome", "error");+ span.record("otel.status_code", "ERROR");+ span.record("error.type", error_type);+ span.record("error", tracing::field::display(error));+}+/// Records the end of one algorithm run: the run counter and duration/// histogram, the `outcome`/`error` fields on `span`, and a warn log when the/// run failed.diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rsindex 896929a48..a7a218104 100644--- a/crates/libsy/tests/observability.rs+++ b/crates/libsy/tests/observability.rs@@ -17,22 +17,27 @@ use std::time::Duration;use async_trait::async_trait;use futures::StreamExt;+use opentelemetry::trace::TracerProvider as _;+use opentelemetry::{Array as OtelArray, Value as OtelValue};use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ResourceMetrics};use opentelemetry_sdk::metrics::{InMemoryMetricExporter, PeriodicReader, SdkMeterProvider};+use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider, SpanData};use parking_lot::Mutex;+use serde_json::json;use tracing::field::{Field, Visit};use tracing::span::{Attributes, Id, Record};use tracing::{Event, Subscriber};+use tracing_opentelemetry::OpenTelemetryLayer;use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt};use tracing_subscriber::registry::LookupSpan;use tracing_subscriber::Layer;use switchyard_libsy::algorithms::{LlmTaskClassifier, TaskClassifierConfig};use switchyard_libsy::{- AggLlmResponse, Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmTarget,+ Algorithm, Context, Decision, Driver, LibsyError, LlmResponse, LlmResponseChunk, LlmTarget,LlmTargetSet, Metadata, Request, Response, RoutedLlmClient, Step, Usage,};-use switchyard_protocol::{text_request, text_response, LlmClientError};+use switchyard_protocol::{text_request, text_response, LlmClientError, StopReason};#[derive(Debug, thiserror::Error)]#[error("{0}")]@@ -92,6 +97,14 @@ impl Visit for FieldVisitor<'_> {fn record_u64(&mut self, field: &Field, value: u64) {self.0.insert(field.name().to_string(), value.to_string());}++ fn record_i64(&mut self, field: &Field, value: i64) {+ self.0.insert(field.name().to_string(), value.to_string());+ }++ fn record_f64(&mut self, field: &Field, value: f64) {+ self.0.insert(field.name().to_string(), value.to_string());+ }}/// Subscriber layer capturing spans (with contextual parents and recorded@@ -146,9 +159,16 @@ where/// Installs the process-global telemetry sinks once: an in-memory OTel metric/// pipeline behind the global meter provider, and the capture layer as the/// global tracing subscriber.-fn telemetry() -> &'static (CaptureStore, InMemoryMetricExporter, SdkMeterProvider) {- static TELEMETRY: OnceLock<(CaptureStore, InMemoryMetricExporter, SdkMeterProvider)> =- OnceLock::new();+type Telemetry = (+ CaptureStore,+ InMemoryMetricExporter,+ SdkMeterProvider,+ InMemorySpanExporter,+ SdkTracerProvider,+);++fn telemetry() -> &'static Telemetry {+ static TELEMETRY: OnceLock<Telemetry> = OnceLock::new();TELEMETRY.get_or_init(|| {let exporter = InMemoryMetricExporter::default();let reader = PeriodicReader::builder(exporter.clone()).build();@@ -156,18 +176,27 @@ fn telemetry() -> &'static (CaptureStore, InMemoryMetricExporter, SdkMeterProvidopentelemetry::global::set_meter_provider(provider.clone());switchyard_libsy::initialize_metrics();+ let span_exporter = InMemorySpanExporter::default();+ let tracer_provider = SdkTracerProvider::builder()+ .with_simple_exporter(span_exporter.clone())+ .build();+ let tracer = tracer_provider.tracer("switchyard-observability-test");let store = CaptureStore::default();- let subscriber = tracing_subscriber::registry().with(CaptureLayer {- store: store.clone(),- });+ let otel_layer: OpenTelemetryLayer<_, _> =+ tracing_opentelemetry::layer().with_tracer(tracer);+ let subscriber = tracing_subscriber::registry()+ .with(CaptureLayer {+ store: store.clone(),+ })+ .with(otel_layer);if tracing::subscriber::set_global_default(subscriber).is_err() {panic!("a global tracing subscriber was already installed in this test binary");}- (store, exporter, provider)+ (store, exporter, provider, span_exporter, tracer_provider)})}-/// The three tests in this file must not overlap because metrics are global.+/// The tests in this file must not overlap because metrics are global./// There is no Rust/cargo-native way of saying this (people use `serial_test` crate) so use a/// lock./// Each file in `tests/` (integration tests) runs as a separate test process, so we are not@@ -343,12 +372,15 @@ impl RoutedLlmClient for UsageClient {_request: Request,decision: Arc<dyn Decision>,) -> Result<Response, switchyard_protocol::LlmClientError> {+ let mut response = text_response(+ Some(decision.selected_model().to_string()),+ "observed response",+ );+ response.id = Some("obs-response-1".to_string());+ response.usage = self.usage.clone();+ response.outputs[0].stop_reason = Some(StopReason::EndTurn);Ok(Response {- llm_response: LlmResponse::Agg(AggLlmResponse {- model: Some(decision.selected_model().to_string()),- usage: self.usage.clone(),- ..AggLlmResponse::default()- }),+ llm_response: LlmResponse::Agg(response),metadata: None,})}@@ -426,10 +458,47 @@ fn find_span(spans: &[SpanRecord], name: &str, field: &str, value: &str) -> Span}}+fn find_otel_span(exporter: &InMemorySpanExporter, name: &str, model: &str) -> SpanData {+ let spans = match exporter.get_finished_spans() {+ Ok(spans) => spans,+ Err(error) => panic!("failed to read exported spans: {error}"),+ };+ match spans.iter().find(|span| {+ span.name == name+ && span.attributes.iter().any(|attribute| {+ attribute.key.as_str() == "gen_ai.request.model"+ && attribute.value.as_str() == model+ })+ }) {+ Some(span) => span.clone(),+ None => {+ let available = spans+ .iter()+ .map(|span| {+ let model = span+ .attributes+ .iter()+ .find(|attribute| attribute.key.as_str() == "gen_ai.request.model")+ .map(|attribute| attribute.value.as_str().into_owned());+ (span.name.to_string(), model)+ })+ .collect::<Vec<_>>();+ panic!("no exported '{name}' span for model {model}; available: {available:?}")+ }+ }+}++fn otel_attribute<'a>(span: &'a SpanData, key: &str) -> Option<&'a OtelValue> {+ span.attributes+ .iter()+ .find(|attribute| attribute.key.as_str() == key)+ .map(|attribute| &attribute.value)+}+#[tokio::test]async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;- let (store, exporter, provider) = telemetry();+ let (store, exporter, provider, span_exporter, _) = telemetry();const ALGO: &str = "obs-success-algo";const MODEL: &str = "obs-success-model";let before = flushed_metrics(exporter, provider);@@ -442,16 +511,20 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lusage: Usage {input_tokens: Some(11),output_tokens: Some(7),- total_tokens: Some(18),+ total_tokens: Some(25),reasoning_tokens: Some(2),- ..Usage::default()+ cache: Usage::cache_details(Some(3), Some(4)),},}) as Arc<dyn RoutedLlmClient>;+ let mut request = request_with_metadata("obs-session-1", "obs-corr-1");+ request.llm_request.sampling.temperature = Some(0.25);+ request.llm_request.sampling.top_p = Some(0.9);+ request.llm_request.sampling.top_k = Some(40);+ request.llm_request.output.max_output_tokens = Some(512);+ request.llm_request.output.response_format = Some(json!({"type": "json_schema"}));+ request.llm_request.reasoning.effort = Some("high".to_string());let (trace, _response) = algo(ALGO, MODEL, Some(client))- .run(- Context::default(),- request_with_metadata("obs-session-1", "obs-corr-1"),- )+ .run(Context::default(), request).await?;assert_eq!(trace.len(), 1);@@ -525,6 +598,21 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lrun_span.fields.get("session_id").map(String::as_str),Some("obs-session-1"));+ assert_eq!(+ run_span.fields.get("session.id").map(String::as_str),+ Some("obs-session-1")+ );+ assert_eq!(+ run_span+ .fields+ .get("switchyard.algorithm")+ .map(String::as_str),+ Some(ALGO)+ );+ assert_eq!(+ run_span.fields.get("switchyard.route").map(String::as_str),+ Some("auto")+ );assert_eq!(run_span.fields.get("correlation_id").map(String::as_str),Some("obs-corr-1")@@ -555,6 +643,49 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lclient_span.fields.get("outcome").map(String::as_str),Some("ok"));+ for (field, value) in [+ ("otel.kind", "client"),+ ("switchyard.algorithm", ALGO),+ ("otel.name", "chat obs-success-model"),+ ("gen_ai.operation.name", "chat"),+ ("gen_ai.request.model", MODEL),+ ("gen_ai.request.temperature", "0.25"),+ ("gen_ai.request.top_p", "0.9"),+ ("gen_ai.request.top_k", "40"),+ ("gen_ai.request.max_tokens", "512"),+ ("gen_ai.request.reasoning.level", "high"),+ ("gen_ai.output.type", "json"),+ ("gen_ai.conversation.id", "obs-session-1"),+ ("gen_ai.response.id", "obs-response-1"),+ ("gen_ai.response.model", MODEL),+ ("gen_ai.usage.input_tokens", "18"),+ ("gen_ai.usage.output_tokens", "7"),+ ("gen_ai.usage.cache_read.input_tokens", "3"),+ ("gen_ai.usage.cache_creation.input_tokens", "4"),+ ("gen_ai.usage.reasoning.output_tokens", "2"),+ ] {+ assert_eq!(+ client_span.fields.get(field).map(String::as_str),+ Some(value),+ "unexpected {field}"+ );+ }+ assert_eq!(client_span.fields.get("gen_ai.request.stream"), None);++ let otel_span = find_otel_span(span_exporter, "chat obs-success-model", MODEL);+ assert!(matches!(+ otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),+ Some(OtelValue::Array(OtelArray::String(reasons)))+ if reasons.len() == 1 && reasons[0].as_str() == "end_turn"+ ));+ assert_eq!(+ otel_attribute(&otel_span, "gen_ai.request.max_tokens"),+ Some(&OtelValue::I64(512))+ );+ assert_eq!(+ otel_attribute(&otel_span, "gen_ai.usage.input_tokens"),+ Some(&OtelValue::I64(18))+ );let call_span = find_span(&spans, "libsy.llm_call", "selected_model", MODEL);assert_eq!(call_span.parent.as_deref(), Some("libsy.run"));@@ -576,7 +707,7 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l);assert_eq!(call_span.fields.get("total_tokens").map(String::as_str),- Some("18")+ Some("25"));assert_eq!(call_span.fields.get("reasoning_tokens").map(String::as_str),@@ -604,10 +735,165 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_lOk(())}+/// A streamed response keeps the client span available until terminal usage arrives.+struct StreamingUsageClient;++#[async_trait]+impl RoutedLlmClient for StreamingUsageClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ decision: Arc<dyn Decision>,+ ) -> Result<Response, LlmClientError> {+ let usage = Usage {+ input_tokens: Some(13),+ output_tokens: Some(5),+ cache: Usage::cache_details(Some(8), None),+ ..Usage::default()+ };+ let chunks = vec![+ Ok(LlmResponseChunk::MessageStart {+ id: Some("obs-stream-response".to_string()),+ model: Some(decision.selected_model().to_string()),+ }),+ Ok(LlmResponseChunk::Usage(usage)),+ Ok(LlmResponseChunk::MessageStop {+ reason: Some("end_turn".to_string()),+ }),+ ];+ Ok(Response {+ llm_response: LlmResponse::Stream(Box::pin(futures::stream::iter(chunks))),+ metadata: None,+ })+ }+}++struct TimeoutClient;++#[async_trait]+impl RoutedLlmClient for TimeoutClient {+ async fn call(+ &self,+ _ctx: Context,+ _request: Request,+ _decision: Arc<dyn Decision>,+ ) -> Result<Response, LlmClientError> {+ Err(LlmClientError::Timeout {+ source: Box::new(TestError("upstream timed out")),+ })+ }+}++#[tokio::test]+async fn streamed_usage_updates_the_client_call_span() -> switchyard_libsy::Result<()> {+ let _guard = serialize_test().lock().await;+ let (store, _, _, span_exporter, _) = telemetry();+ const ALGO: &str = "obs-stream-algo";+ const MODEL: &str = "obs-stream-model";+ let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;+ let mut request = request_with_metadata("obs-stream-session", "obs-stream-corr");+ request.llm_request.stream = true;+ let (_, response) = algo(ALGO, MODEL, Some(client))+ .run(Context::default(), request)+ .await?;+ let LlmResponse::Stream(mut stream) = response.llm_response else {+ return Err(test_error("expected a streamed response"));+ };+ while let Some(item) = stream.next().await {+ if let Err(error) = item {+ panic!("unexpected stream error: {error}");+ }+ }++ let spans = store.spans();+ let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);+ for (field, value) in [+ ("otel.name", "chat obs-stream-model"),+ ("gen_ai.request.stream", "true"),+ ("gen_ai.response.id", "obs-stream-response"),+ ("gen_ai.response.model", MODEL),+ ("gen_ai.usage.input_tokens", "21"),+ ("gen_ai.usage.output_tokens", "5"),+ ("gen_ai.usage.cache_read.input_tokens", "8"),+ ] {+ assert_eq!(+ client_span.fields.get(field).map(String::as_str),+ Some(value),+ "unexpected {field}"+ );+ }+ let otel_span = find_otel_span(span_exporter, "chat obs-stream-model", MODEL);+ assert!(matches!(+ otel_attribute(&otel_span, "gen_ai.response.finish_reasons"),+ Some(OtelValue::Array(OtelArray::String(reasons)))+ if reasons.len() == 1 && reasons[0].as_str() == "end_turn"+ ));+ Ok(())+}++#[tokio::test]+async fn dropped_stream_records_cancelled_outcome() -> switchyard_libsy::Result<()> {+ let _guard = serialize_test().lock().await;+ let (store, _, _, _, _) = telemetry();+ const ALGO: &str = "obs-cancelled-stream-algo";+ const MODEL: &str = "obs-cancelled-stream-model";+ let client = Arc::new(StreamingUsageClient) as Arc<dyn RoutedLlmClient>;+ let mut request = request_with_metadata("obs-cancelled-session", "obs-cancelled-corr");+ request.llm_request.stream = true;+ let (_, response) = algo(ALGO, MODEL, Some(client))+ .run(Context::default(), request)+ .await?;+ let LlmResponse::Stream(stream) = response.llm_response else {+ return Err(test_error("expected a streamed response"));+ };+ drop(stream);++ let spans = store.spans();+ let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);+ assert_eq!(+ client_span.fields.get("outcome").map(String::as_str),+ Some("cancelled")+ );+ Ok(())+}++#[tokio::test]+async fn typed_client_failure_records_semantic_error_type() {+ let _guard = serialize_test().lock().await;+ let (store, _, _, _, _) = telemetry();+ const ALGO: &str = "obs-timeout-algo";+ const MODEL: &str = "obs-timeout-model";+ let result = algo(+ ALGO,+ MODEL,+ Some(Arc::new(TimeoutClient) as Arc<dyn RoutedLlmClient>),+ )+ .run(+ Context::default(),+ request_with_metadata("obs-timeout-session", "obs-timeout-corr"),+ )+ .await;+ assert!(matches!(+ result,+ Err(LibsyError::ClientCall {+ source: LlmClientError::Timeout { .. },+ ..+ })+ ));++ let spans = store.spans();+ let client_span = find_span(&spans, "libsy.client_call", "selected_model", MODEL);+ assert_eq!(+ client_span.fields.get("error.type").map(String::as_str),+ Some("timeout")+ );+}+#[tokio::test]async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;- let (store, exporter, provider) = telemetry();+ let (store, exporter, provider, _, _) = telemetry();const ALGO: &str = "obs-failure-algo";const MODEL: &str = "obs-failure-model";let before = flushed_metrics(exporter, provider);@@ -729,7 +1015,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::#[tokio::test]async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_libsy::Result<()> {let _guard = serialize_test().lock().await;- let (_store, exporter, provider) = telemetry();+ let (_store, exporter, provider, _, _) = telemetry();let before = flushed_metrics(exporter, provider);let total_requests_before =u64_gauge_value(&before, "switchyard.total_requests").unwrap_or_default();diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rsindex 9e3ee3028..be00ffc59 100644--- a/crates/switchyard-server/src/lib.rs+++ b/crates/switchyard-server/src/lib.rs@@ -35,7 +35,7 @@ use libsy::{};use serde_json::{json, Value};use tokio::net::{TcpListener, TcpSocket};-use tracing::Level;+use tracing::{Instrument, Level};use switchyard_translation::{decode_request, WireFormat};@@ -418,6 +418,19 @@ async fn handle_endpoint(headers: HeaderMap,body: std::result::Result<Json<Value>, JsonRejection>,wire_format: WireFormat,+) -> Response {+ let span = observability::request_span(&headers);+ handle_endpoint_inner(state, started, headers, body, wire_format)+ .instrument(span)+ .await+}++async fn handle_endpoint_inner(+ state: ServerState,+ started: RequestStart,+ headers: HeaderMap,+ body: std::result::Result<Json<Value>, JsonRejection>,+ wire_format: WireFormat,) -> Response {let metadata = metadata_from_headers(&headers);let request_log = RequestLogContext {diff --git a/crates/switchyard-server/src/observability.rs b/crates/switchyard-server/src/observability.rsindex 6f9735bbf..fe3036cf2 100644--- a/crates/switchyard-server/src/observability.rs+++ b/crates/switchyard-server/src/observability.rs@@ -6,9 +6,13 @@use std::env;use std::sync::OnceLock;+use axum::http::HeaderMap;+use opentelemetry::propagation::{Extractor, TextMapPropagator};use opentelemetry::trace::TracerProvider as _;+use opentelemetry_sdk::propagation::TraceContextPropagator;use opentelemetry_sdk::trace::SdkTracerProvider;use opentelemetry_sdk::Resource;+use tracing_opentelemetry::OpenTelemetrySpanExt;use tracing_subscriber::layer::SubscriberExt as _;use tracing_subscriber::util::SubscriberInitExt as _;use tracing_subscriber::{EnvFilter, Layer as _};@@ -44,6 +48,30 @@ pub fn flush_observability() {metrics::flush();}+/// Creates the server request span with any incoming W3C trace context as its parent.+pub(crate) fn request_span(headers: &HeaderMap) -> tracing::Span {+ let parent = TraceContextPropagator::new().extract(&HeaderExtractor(headers));+ let span = tracing::info_span!(+ target: "switchyard_server",+ "switchyard.request",+ otel.kind = "server",+ );+ let _ = span.set_parent(parent);+ span+}++struct HeaderExtractor<'a>(&'a HeaderMap);++impl Extractor for HeaderExtractor<'_> {+ fn get(&self, key: &str) -> Option<&str> {+ self.0.get(key).and_then(|value| value.to_str().ok())+ }++ fn keys(&self) -> Vec<&str> {+ self.0.keys().map(|name| name.as_str()).collect()+ }+}+pub(crate) fn otlp_enabled(signal: &str) -> bool {if env_var_is_true("OTEL_SDK_DISABLED") {return false;@@ -130,3 +158,43 @@ fn build_tracer_provider() -> Result<SdkTracerProvider, String> {fn env_var_is_true(name: &str) -> bool {env::var(name).is_ok_and(|value| matches!(value.to_ascii_lowercase().as_str(), "true" | "1"))}++#[cfg(test)]+mod tests {+ use axum::http::{HeaderMap, HeaderValue};+ use opentelemetry::trace::{TraceContextExt, TracerProvider as _};+ use opentelemetry_sdk::trace::SdkTracerProvider;+ use tracing_opentelemetry::OpenTelemetrySpanExt as _;+ use tracing_subscriber::layer::SubscriberExt as _;++ use super::request_span;++ #[test]+ fn request_span_continues_incoming_w3c_trace_context() {+ let provider = SdkTracerProvider::builder().build();+ let tracer = provider.tracer("request-span-test");+ let subscriber =+ tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer));+ let mut headers = HeaderMap::new();+ headers.insert(+ "traceparent",+ HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),+ );+ headers.insert(+ "tracestate",+ HeaderValue::from_static("vendor=opaque-value"),+ );++ tracing::subscriber::with_default(subscriber, || {+ let span = request_span(&headers);+ let context = span.context();+ let current = context.span();+ let span_context = current.span_context();+ assert_eq!(+ span_context.trace_id().to_string(),+ "4bf92f3577b34da6a3ce929d0e0e4736"+ );+ assert_eq!(span_context.trace_state().header(), "vendor=opaque-value");+ });+ }+}