Skip to content

Switchyard lets LLM applications route traffic across models and providers while preserving native OpenAI and Anthropic API compatibility - enabling flexible model selection, benchmarking, and cost/performance optimization.

Rust

fix(libsy): client disconnect logging (#555)

Greg Clark77af9b6be03b
diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs
index 914fb6c8a..77150399c 100644
--- a/crates/switchyard-server/src/lib.rs
+++ b/crates/switchyard-server/src/lib.rs
@@ -63,6 +63,9 @@ pub const DEFAULT_MAX_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024;
const HEADER_SELECTED_MODEL: &str = "x-model-router-selected-model";
const MAX_ROUTING_HEADER_VALUE_LEN: usize = 512;
+/// Non-standard status used only in logs and metrics for a request whose
+/// downstream client disconnected before any response was written.
+const CLIENT_CLOSED_REQUEST: u16 = 499;
const STARTUP_BANNER_ART: &str = include_str!("../assets/startup_banner.txt");
/// Error returned while configuring or running the server.
@@ -693,6 +696,12 @@ async fn handle_endpoint_inner(
correlation_id: metadata.correlation_id.clone(),
};
+ // Axum drops this future when the downstream client disconnects before the
+ // response is written, which cancels the run and its upstream calls. The guard
+ // keeps that terminal state observable: the `emit` path below never runs for an
+ // abandoned buffered request.
+ let mut request_log = RequestLogGuard(Some(request_log));
+
let response = match llm_json_body(body) {
Ok(body) => {
handle_llm_request(
@@ -847,6 +856,29 @@ async fn handle_llm_request(
response
}
+/// Holds the request log context until the request reaches a terminal state.
+/// Dropping the guard with the context still in place means the handler future was
+/// cancelled — the client left before a response existed — so the run is reported
+/// as abandoned rather than silently disappearing.
+struct RequestLogGuard(Option<RequestLogContext>);
+
+impl RequestLogGuard {
+ // Emits the normal terminal event and disarms the cancellation path.
+ fn emit(&mut self, response: &Response) {
+ if let Some(context) = self.0.take() {
+ context.emit(response);
+ }
+ }
+}
+
+impl Drop for RequestLogGuard {
+ fn drop(&mut self) {
+ if let Some(context) = self.0.take() {
+ context.emit_cancelled();
+ }
+ }
+}
+
// Request metadata held until the terminal response determines the event level.
struct RequestLogContext {
started: Instant,
@@ -900,6 +932,26 @@ impl RequestLogContext {
_ => emit!(Level::INFO, "LLM request handled"),
}
}
+
+ // Terminal event for a request whose client left before a response was written.
+ // No model served it, so there is no selected model and no response status.
+ fn emit_cancelled(self) {
+ metrics::record_client_disconnect();
+ tracing::event!(
+ target: "switchyard_server::request",
+ Level::WARN,
+ wire_format = %self.wire_format,
+ status = CLIENT_CLOSED_REQUEST,
+ requested_model = self.requested_model.as_deref().unwrap_or(""),
+ selected_model = "",
+ streaming = self.streaming,
+ session_id = self.session_id.as_deref().unwrap_or(""),
+ correlation_id = self.correlation_id.as_deref().unwrap_or(""),
+ handling_duration_ms = self.started.elapsed().as_secs_f64() * 1_000.0,
+ error = "client disconnected before a response was written",
+ "LLM request cancelled"
+ );
+ }
}
fn request_log_level(status: StatusCode) -> Level {
@@ -1563,6 +1615,146 @@ mod tests {
request.abort();
}
+ #[derive(Clone)]
+ struct CancelTestState {
+ started: Arc<Notify>,
+ cancelled: Arc<Notify>,
+ }
+
+ // Signals when the handler future is dropped, standing in for the in-flight
+ // upstream call that a cancelled run releases.
+ struct CancelSentinel(Arc<Notify>);
+
+ impl Drop for CancelSentinel {
+ fn drop(&mut self) {
+ self.0.notify_one();
+ }
+ }
+
+ async fn never_responds(State(state): State<CancelTestState>) -> &'static str {
+ let _sentinel = CancelSentinel(state.cancelled.clone());
+ state.started.notify_one();
+ std::future::pending::<()>().await;
+ "unreachable"
+ }
+
+ // A buffered request produces no response until the run finishes, so nothing is
+ // written to the socket to reveal that the client left. The server must still drop
+ // the handler future on disconnect, otherwise upstream work outlives its client.
+ #[tokio::test]
+ async fn client_disconnect_cancels_a_buffered_handler() {
+ let state = CancelTestState {
+ started: Arc::new(Notify::new()),
+ cancelled: Arc::new(Notify::new()),
+ };
+ let router = Router::new()
+ .route("/", post(never_responds))
+ .with_state(state.clone());
+ let listener = bind_tcp_listener("127.0.0.1:0".parse().expect("valid address"), 16)
+ .expect("listener binds");
+ let addr = listener.local_addr().expect("listener has an address");
+ let (shutdown, shutdown_receiver) = oneshot::channel();
+ let server = tokio::spawn(serve(
+ listener,
+ router,
+ Duration::from_millis(25),
+ async move {
+ let _ = shutdown_receiver.await;
+ },
+ ));
+
+ let mut stream = tokio::net::TcpStream::connect(addr)
+ .await
+ .expect("client connects");
+ stream
+ .write_all(b"POST / HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}")
+ .await
+ .expect("client sends a complete request");
+ state.started.notified().await;
+
+ // The client goes away mid-run, before any response byte exists.
+ drop(stream);
+
+ tokio::time::timeout(Duration::from_secs(5), state.cancelled.notified())
+ .await
+ .expect("handler future is dropped when the client disconnects");
+
+ let _ = shutdown.send(());
+ let _ = tokio::time::timeout(Duration::from_secs(5), server).await;
+ }
+
+ // Collects the terminal request events emitted while it is the active subscriber.
+ #[derive(Clone, Default)]
+ struct CapturedEvents(Arc<Mutex<Vec<(Level, String)>>>);
+
+ impl tracing_subscriber::Layer<tracing_subscriber::Registry> for CapturedEvents {
+ fn on_event(
+ &self,
+ event: &tracing::Event<'_>,
+ _ctx: tracing_subscriber::layer::Context<'_, tracing_subscriber::Registry>,
+ ) {
+ if event.metadata().target() != "switchyard_server::request" {
+ return;
+ }
+ let mut message = String::new();
+ event.record(
+ &mut |field: &tracing::field::Field, value: &dyn std::fmt::Debug| {
+ if field.name() == "message" {
+ message = format!("{value:?}");
+ }
+ },
+ );
+ self.0.lock().push((*event.metadata().level(), message));
+ }
+ }
+
+ fn request_log_context() -> RequestLogContext {
+ RequestLogContext {
+ started: Instant::now(),
+ wire_format: WireFormat::OpenAiChat,
+ requested_model: Some("switchyard/route".to_string()),
+ streaming: false,
+ session_id: Some("session".to_string()),
+ correlation_id: None,
+ }
+ }
+
+ fn captured_events(run: impl FnOnce()) -> Vec<(Level, String)> {
+ use tracing_subscriber::layer::SubscriberExt;
+
+ let captured = CapturedEvents::default();
+ let subscriber = tracing_subscriber::registry().with(captured.clone());
+ tracing::subscriber::with_default(subscriber, run);
+ captured.0.lock().clone()
+ }
+
+ // A dropped handler future is the only trace an abandoned buffered request leaves,
+ // so the guard has to turn it into a terminal event rather than logging nothing.
+ #[test]
+ fn dropping_the_request_log_guard_reports_a_cancelled_request() {
+ let events = captured_events(|| {
+ drop(RequestLogGuard(Some(request_log_context())));
+ });
+
+ assert_eq!(events.len(), 1);
+ assert_eq!(events[0].0, Level::WARN);
+ assert!(events[0].1.contains("cancelled"), "{:?}", events[0].1);
+ }
+
+ // A request that reached a response is not a cancellation, even though the guard
+ // is still dropped afterwards.
+ #[test]
+ fn an_answered_request_reports_only_its_response() {
+ let events = captured_events(|| {
+ let mut guard = RequestLogGuard(Some(request_log_context()));
+ guard.emit(&StatusCode::OK.into_response());
+ });
+
+ assert_eq!(events.len(), 1);
+ assert_eq!(events[0].0, Level::INFO);
+ assert!(events[0].1.contains("handled"), "{:?}", events[0].1);
+ }
+
// Terminal request severity follows HTTP status instead of error-path bookkeeping.
#[test]
fn request_log_level_follows_http_status() {
diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rs
index 41bd50f5c..1f05c6f75 100644
--- a/crates/switchyard-server/src/metrics.rs
+++ b/crates/switchyard-server/src/metrics.rs
@@ -12,6 +12,10 @@ use switchyard_llm_client::metrics::{http_outcome_label, http_status_code_label}
pub(crate) const CONTENT_TYPE: &str = "text/plain; version=0.0.4; charset=utf-8";
+/// Outcome label for a request the client abandoned. Kept separate from the
+/// status-derived outcomes so cancelled work is never counted as a served response.
+const CLIENT_DISCONNECTED_OUTCOME: &str = "client_disconnected";
+
/// Bucket boundaries for `switchyard.routing_overhead_ms`.
/// Need a broad range because some algos call an LLM (classifier), and some
/// do very little (passthrough).
@@ -124,7 +128,12 @@ fn seed_outcome_metrics() {
}
let client_responses = meter.u64_counter("switchyard.client_responses").build();
- for outcome in ["success", "retryable_error", "other_error"] {
+ for outcome in [
+ "success",
+ "retryable_error",
+ "other_error",
+ CLIENT_DISCONNECTED_OUTCOME,
+ ] {
client_responses.add(0, &[KeyValue::new("outcome", outcome)]);
}
meter
@@ -133,6 +142,15 @@ fn seed_outcome_metrics() {
.add(0, &[]);
}
+/// Records a request whose downstream client disconnected before a response was
+/// written. The run and its upstream calls were cancelled, so no status exists.
+pub(crate) fn record_client_disconnect() {
+ global::meter("switchyard")
+ .u64_counter("switchyard.client_responses")
+ .build()
+ .add(1, &[KeyValue::new("outcome", CLIENT_DISCONNECTED_OUTCOME)]);
+}
+
/// Records the final status returned by an LLM-serving route.
pub(crate) fn record_client_response(status: u16) {
global::meter("switchyard")