Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

feat(server): add structured request logging (#159)

nachiketb-nvidia063ca463ddce
diff --git a/Cargo.lock b/Cargo.lock
index d0082b8c0..514bff6a0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -950,6 +950,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+[[package]]
+name = "matchers"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
+dependencies = [
+ "regex-automata",
+]
+
[[package]]
name = "matchit"
version = "0.8.4"
@@ -1852,6 +1861,7 @@ dependencies = [
"toml",
"tower",
"tracing",
+ "tracing-subscriber",
]
[[package]]
@@ -2199,10 +2209,14 @@ version = "0.3.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
dependencies = [
+ "matchers",
"nu-ansi-term",
+ "once_cell",
+ "regex-automata",
"sharded-slab",
"smallvec",
"thread_local",
+ "tracing",
"tracing-core",
"tracing-log",
]
diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs
index 5d7d10ead..0b21ee6f6 100644
--- a/crates/libsy/src/observability.rs
+++ b/crates/libsy/src/observability.rs
@@ -231,11 +231,11 @@ pub(crate) fn record_llm_call(
}
/// Records one published routing decision: the decision counter plus a
-/// structured info log carrying the decision's reasoning.
+/// structured debug event carrying the decision's reasoning.
pub(crate) fn record_decision(ctx: &Context, decision: &dyn Decision) {
let algorithm = algorithm_label(ctx);
let selected_model = decision.selected_model();
- tracing::info!(
+ tracing::debug!(
target: SCOPE,
algorithm,
selected_model,
diff --git a/crates/libsy/tests/observability.rs b/crates/libsy/tests/observability.rs
index 6af79bad0..521af72e4 100644
--- a/crates/libsy/tests/observability.rs
+++ b/crates/libsy/tests/observability.rs
@@ -53,6 +53,7 @@ struct SpanRecord {
#[derive(Clone, Debug, Default)]
struct EventRecord {
target: String,
+ level: String,
fields: BTreeMap<String, String>,
}
@@ -134,6 +135,7 @@ where
event.record(&mut FieldVisitor(&mut fields));
self.store.events.lock().push(EventRecord {
target: event.metadata().target().to_string(),
+ level: event.metadata().level().to_string(),
fields,
});
}
@@ -492,11 +494,12 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
Some("2")
);
- // Structured log: the published decision with its reasoning.
+ // Structured debug event: the published decision with its reasoning.
let events = store.events();
assert!(
events.iter().any(|event| {
event.target == "libsy"
+ && event.level == "DEBUG"
&& event.fields.get("selected_model").map(String::as_str) == Some(MODEL)
&& event
.fields
@@ -586,11 +589,12 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
Some("error")
);
- // Structured logs: a warn per failed call and per failed run.
+ // Structured logs warn once for the failed call and failed run.
let events = store.events();
assert!(
events.iter().any(|event| {
event.target == "libsy"
+ && event.level == "WARN"
&& event.fields.get("selected_model").map(String::as_str) == Some(MODEL)
&& event
.fields
@@ -602,6 +606,7 @@ async fn failed_call_records_error_outcome_and_warn_logs() -> switchyard_libsy::
assert!(
events.iter().any(|event| {
event.target == "libsy"
+ && event.level == "WARN"
&& event.fields.get("algorithm").map(String::as_str) == Some(ALGO)
&& event
.fields
diff --git a/crates/switchyard-server/Cargo.toml b/crates/switchyard-server/Cargo.toml
index 9bb4010e0..f1044b346 100644
--- a/crates/switchyard-server/Cargo.toml
+++ b/crates/switchyard-server/Cargo.toml
@@ -30,6 +30,7 @@ serde_json.workspace = true
switchyard-translation.workspace = true
tokio.workspace = true
tracing.workspace = true
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
[dev-dependencies]
async-trait.workspace = true
diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md
index 611f7b207..4aa8fb47d 100644
--- a/crates/switchyard-server/README.md
+++ b/crates/switchyard-server/README.md
@@ -47,6 +47,11 @@ export API_KEY="..."
cargo run -p switchyard-server -- --config routes.toml
```
+The server logs exactly one structured terminal event per LLM request: successful responses at
+`INFO`, 4xx responses at `WARN`, and 5xx responses at `ERROR`. Set
+`RUST_LOG=switchyard_server=debug,libsy=debug` to include routing decisions and nested failure
+details. A streaming failure is logged separately because it can occur after the response starts.
+
Target and route table names are local references. A target's `id` is the exact model ID sent
upstream, and a route's `id` is the model clients send to select that algorithm.
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 2720e9235..dbcb663c5 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -14,7 +14,7 @@ use std::fmt::{Display, Formatter};
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
use axum::extract::{rejection::JsonRejection, State};
use axum::http::header::CONTENT_TYPE;
@@ -26,6 +26,7 @@ use axum_server::tls_rustls::RustlsConfig;
use libsy::{Algorithm, Context, Decision, LibsyError, LlmClientError, Metadata, Request};
use serde_json::{json, Value};
use tokio::net::{TcpListener, TcpSocket};
+use tracing::Level;
use switchyard_translation::{decode_request, WireFormat};
@@ -258,11 +259,32 @@ async fn handle_endpoint(
body: std::result::Result<Json<Value>, JsonRejection>,
wire_format: WireFormat,
) -> Response {
- let body = match llm_json_body(body) {
- Ok(body) => body,
- Err(message) => return invalid_body_error(message),
+ let metadata = metadata_from_headers(&headers);
+ let request_log = RequestLogContext {
+ started: Instant::now(),
+ wire_format,
+ requested_model: body
+ .as_ref()
+ .ok()
+ .and_then(|body| body.0.get("model"))
+ .and_then(Value::as_str)
+ .map(str::to_string),
+ streaming: body
+ .as_ref()
+ .ok()
+ .and_then(|body| body.0.get("stream"))
+ .and_then(Value::as_bool)
+ .unwrap_or(false),
+ session_id: metadata.session_id.clone(),
+ correlation_id: metadata.correlation_id.clone(),
};
- handle_llm_request(state, headers, body, wire_format).await
+
+ let response = match llm_json_body(body) {
+ Ok(body) => handle_llm_request(state, metadata, body, wire_format).await,
+ Err(message) => invalid_body_error(message),
+ };
+ request_log.emit(&response);
+ response
}
fn llm_json_body(
@@ -277,7 +299,7 @@ fn llm_json_body(
async fn handle_llm_request(
state: ServerState,
- headers: HeaderMap,
+ metadata: Metadata,
body: Value,
wire_format: WireFormat,
) -> Response {
@@ -309,7 +331,7 @@ async fn handle_llm_request(
let request = Request {
llm_request,
raw_request: Some(body),
- metadata: Some(metadata_from_headers(&headers)),
+ metadata: Some(metadata),
};
let (trace, response) = match algorithm.run(Context::default(), request).await {
Ok(result) => result,
@@ -326,6 +348,71 @@ async fn handle_llm_request(
response
}
+// Request metadata held until the terminal response determines the event level.
+struct RequestLogContext {
+ started: Instant,
+ wire_format: WireFormat,
+ requested_model: Option<String>,
+ streaming: bool,
+ session_id: Option<String>,
+ correlation_id: Option<String>,
+}
+
+// Error text carried separately so terminal logging never consumes an HTTP body.
+#[derive(Clone)]
+struct RequestLogError(String);
+
+impl RequestLogContext {
+ fn emit(self, response: &Response) {
+ let selected_model = response
+ .headers()
+ .get(HEADER_SELECTED_MODEL)
+ .and_then(|value| value.to_str().ok())
+ .unwrap_or("");
+ let duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0;
+ let error = response
+ .extensions()
+ .get::<RequestLogError>()
+ .map(|error| error.0.as_str())
+ .unwrap_or("");
+
+ macro_rules! emit {
+ ($level:expr, $message:literal) => {
+ tracing::event!(
+ target: "switchyard_server::request",
+ $level,
+ wire_format = %self.wire_format,
+ status = response.status().as_u16(),
+ requested_model = self.requested_model.as_deref().unwrap_or(""),
+ selected_model,
+ streaming = self.streaming,
+ session_id = self.session_id.as_deref().unwrap_or(""),
+ correlation_id = self.correlation_id.as_deref().unwrap_or(""),
+ handling_duration_ms = duration_ms,
+ error,
+ $message
+ )
+ };
+ }
+
+ match request_log_level(response.status()) {
+ Level::ERROR => emit!(Level::ERROR, "LLM request failed"),
+ Level::WARN => emit!(Level::WARN, "LLM request failed"),
+ _ => emit!(Level::INFO, "LLM request handled"),
+ }
+ }
+}
+
+fn request_log_level(status: StatusCode) -> Level {
+ if status.is_server_error() {
+ Level::ERROR
+ } else if status.is_success() {
+ Level::INFO
+ } else {
+ Level::WARN
+ }
+}
+
fn metadata_from_headers(headers: &HeaderMap) -> Metadata {
let headers = normalized_headers(headers);
let mut metadata = Metadata::from_headers(&headers);
@@ -448,17 +535,20 @@ fn error_response(
error_type: &'static str,
code: &'static str,
) -> Response {
- (
+ let message = message.into();
+ let mut response = (
status,
Json(json!({
"error": {
- "message": message.into(),
+ "message": message.clone(),
"type": error_type,
"code": code,
}
})),
)
- .into_response()
+ .into_response();
+ response.extensions_mut().insert(RequestLogError(message));
+ response
}
async fn models(State(state): State<ServerState>) -> Json<Value> {
@@ -547,3 +637,38 @@ fn host_for_url(ip: std::net::IpAddr) -> String {
std::net::IpAddr::V6(ip) => format!("[{ip}]"),
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ // Terminal request severity follows HTTP status instead of error-path bookkeeping.
+ #[test]
+ fn request_log_level_follows_http_status() {
+ assert_eq!(request_log_level(StatusCode::OK), Level::INFO);
+ assert_eq!(request_log_level(StatusCode::BAD_REQUEST), Level::WARN);
+ assert_eq!(
+ request_log_level(StatusCode::INTERNAL_SERVER_ERROR),
+ Level::ERROR
+ );
+ }
+
+ // Canonical error text remains available without consuming the response body.
+ #[test]
+ fn error_response_carries_request_log_error() {
+ let response = error_response(
+ StatusCode::BAD_REQUEST,
+ "invalid request",
+ "invalid_request_error",
+ "invalid_request_error",
+ );
+
+ assert_eq!(
+ response
+ .extensions()
+ .get::<RequestLogError>()
+ .map(|error| error.0.as_str()),
+ Some("invalid request")
+ );
+ }
+}
diff --git a/crates/switchyard-server/src/main.rs b/crates/switchyard-server/src/main.rs
index c1f2b7965..e111f6fcf 100644
--- a/crates/switchyard-server/src/main.rs
+++ b/crates/switchyard-server/src/main.rs
@@ -5,10 +5,18 @@
use std::process::ExitCode;
+use tracing_subscriber::EnvFilter;
+
mod cli;
+const DEFAULT_LOG_FILTER: &str = "switchyard_server=info";
+
#[tokio::main(flavor = "multi_thread")]
async fn main() -> ExitCode {
+ if let Err(error) = init_logging() {
+ eprintln!("failed to initialize logging: {error}");
+ return ExitCode::FAILURE;
+ }
match cli::run(cli::ServerArgs::parse_args()).await {
Ok(()) => ExitCode::SUCCESS,
Err(error) => {
@@ -17,3 +25,14 @@ async fn main() -> ExitCode {
}
}
}
+
+fn init_logging() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+ let filter =
+ EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(DEFAULT_LOG_FILTER));
+ tracing_subscriber::fmt()
+ .with_env_filter(filter)
+ .with_ansi(false)
+ .with_writer(std::io::stderr)
+ .try_init()?;
+ Ok(())
+}