Skip to content

🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.

feat(ai-gateway): log per-connection socket writes and output completeness (#19154)

Tobias Wochinger2b8c5c9d37fc
diff --git a/ai-gateway/src/observability/http.rs b/ai-gateway/src/observability/http.rs
index 5f5399952f..774eb84289 100644
--- a/ai-gateway/src/observability/http.rs
+++ b/ai-gateway/src/observability/http.rs
@@ -3,7 +3,7 @@ use std::time::{Duration, Instant};
use axum::{
Router,
body::Body,
- extract::MatchedPath,
+ extract::{ConnectInfo, MatchedPath},
http::{HeaderMap, Request, StatusCode},
middleware::{self, Next},
response::Response,
@@ -14,6 +14,7 @@ use tracing::Span;
use tracing_opentelemetry::OpenTelemetrySpanExt;
use super::request::{ErrorOrigin, RequestObservation, Summary};
+use crate::server::ConnectionId;
/// Instrument application routes; mount health probes outside this router.
pub fn instrument(router: Router) -> Router {
@@ -23,7 +24,10 @@ pub fn instrument(router: Router) -> Router {
TraceLayer::new_for_http()
.make_span_with(|request: &Request<Body>| {
let (method, route) = request_labels(request);
- let span = tracing::info_span!(parent: None, "http.server", otel.name = %format_args!("{method} {route}"), otel.kind = "server", http.request.method = method, http.route = route, http.request.body.size = tracing::field::Empty, http.response.status_code = tracing::field::Empty, otel.status_code = tracing::field::Empty, otel.status_description = tracing::field::Empty, error.type = tracing::field::Empty, error.origin = tracing::field::Empty, gateway.request.id = tracing::field::Empty, gateway.client.request.id = tracing::field::Empty, langfuse.organization.id = tracing::field::Empty, langfuse.project.id = tracing::field::Empty, gateway.provider = tracing::field::Empty, gateway.provider.connection.id = tracing::field::Empty, gateway.provider.connection.name = tracing::field::Empty, gateway.upstream.model = tracing::field::Empty, provider_request_id = tracing::field::Empty, gateway.outcome = tracing::field::Empty, gateway.first_byte_ms = tracing::field::Empty);
+ let span = tracing::info_span!(parent: None, "http.server", otel.name = %format_args!("{method} {route}"), otel.kind = "server", http.request.method = method, http.route = route, http.request.body.size = tracing::field::Empty, http.response.status_code = tracing::field::Empty, otel.status_code = tracing::field::Empty, otel.status_description = tracing::field::Empty, error.type = tracing::field::Empty, error.origin = tracing::field::Empty, gateway.request.id = tracing::field::Empty, gateway.connection.id = tracing::field::Empty, gateway.client.request.id = tracing::field::Empty, langfuse.organization.id = tracing::field::Empty, langfuse.project.id = tracing::field::Empty, gateway.provider = tracing::field::Empty, gateway.provider.connection.id = tracing::field::Empty, gateway.provider.connection.name = tracing::field::Empty, gateway.upstream.model = tracing::field::Empty, provider_request_id = tracing::field::Empty, gateway.outcome = tracing::field::Empty, gateway.first_byte_ms = tracing::field::Empty);
+ if let Some(ConnectInfo(ConnectionId(id))) = request.extensions().get::<ConnectInfo<ConnectionId>>() {
+ span.record("gateway.connection.id", id.as_str());
+ }
// Operational tracing has no caller or ambient trace context.
let _ = span.set_parent(Context::new());
span
diff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rs
index f1a774d276..2a511a2e2d 100644
--- a/ai-gateway/src/observability/http/tests.rs
+++ b/ai-gateway/src/observability/http/tests.rs
@@ -714,6 +714,10 @@ fn assert_request_correlation(
assert_eq!(event[key], expected, "{event}");
}
}
+ if event["message"] == "gateway execution finished" {
+ assert_eq!(event["outcome"], "complete", "{event}");
+ assert_eq!(event["output_complete"], true, "{event}");
+ }
}
for (name, headers, payload) in observed {
assert_eq!(
diff --git a/ai-gateway/src/observability/logs.rs b/ai-gateway/src/observability/logs.rs
index 8097079ba8..8df0a2d339 100644
--- a/ai-gateway/src/observability/logs.rs
+++ b/ai-gateway/src/observability/logs.rs
@@ -15,8 +15,9 @@ use tracing_subscriber::{
};
/// Span field and its top-level log key. All are recorded on the server span.
-const CORRELATION_FIELDS: [(&str, &str); 7] = [
+const CORRELATION_FIELDS: [(&str, &str); 8] = [
("gateway.request.id", "request_id"),
+ ("gateway.connection.id", "connection_id"),
("langfuse.organization.id", "organization_id"),
("langfuse.project.id", "project_id"),
("gateway.provider", "provider"),
diff --git a/ai-gateway/src/observability/metrics.rs b/ai-gateway/src/observability/metrics.rs
index fad50b911c..b7e4df3d9f 100644
--- a/ai-gateway/src/observability/metrics.rs
+++ b/ai-gateway/src/observability/metrics.rs
@@ -276,7 +276,7 @@ pub(crate) fn execution_finished(facts: &InferenceFacts) {
let context = tracing::Span::current().context();
let context_span = context.span();
let ids = context_span.span_context();
- tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), outcome, duration_ms = facts.duration_ms, "gateway execution finished");
+ tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), outcome, duration_ms = facts.duration_ms, output_complete = facts.inference.output_complete, "gateway execution finished");
}
/// Marks the server span failed when the provider body breaks after headers were
diff --git a/ai-gateway/src/observability/mod.rs b/ai-gateway/src/observability/mod.rs
index 53936a7f64..8a35665add 100644
--- a/ai-gateway/src/observability/mod.rs
+++ b/ai-gateway/src/observability/mod.rs
@@ -1,6 +1,6 @@
mod client;
mod http;
-mod logs;
+pub(crate) mod logs;
mod metrics;
mod request;
diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rs
index 616f59cc8e..ab7aa6c61f 100644
--- a/ai-gateway/src/server.rs
+++ b/ai-gateway/src/server.rs
@@ -12,6 +12,10 @@ use axum::{Json, Router, extract::State, http::StatusCode, routing::get};
use serde::Serialize;
use tokio::{net::TcpListener, sync::oneshot};
+mod connection;
+
+pub(crate) use connection::ConnectionId;
+
#[derive(Clone, Default)]
pub struct GatewayLifecycleState {
ready: Arc<AtomicBool>,
@@ -85,19 +89,22 @@ pub async fn serve(
) -> io::Result<()> {
let (draining_tx, draining_rx) = oneshot::channel();
let shutdown_state = state.clone();
- let server = axum::serve(listener, app)
- .with_graceful_shutdown(async move {
- shutdown.await;
- shutdown_state.ready.store(false, Ordering::Release);
- *shutdown_state
- .drain_deadline
- .lock()
- .unwrap_or_else(PoisonError::into_inner) =
- Some(tokio::time::Instant::now() + shutdown_timeout);
- tracing::info!("gateway draining");
- let _ = draining_tx.send(());
- })
- .into_future();
+ let server = axum::serve(
+ connection::ObservedListener(listener),
+ app.into_make_service_with_connect_info::<ConnectionId>(),
+ )
+ .with_graceful_shutdown(async move {
+ shutdown.await;
+ shutdown_state.ready.store(false, Ordering::Release);
+ *shutdown_state
+ .drain_deadline
+ .lock()
+ .unwrap_or_else(PoisonError::into_inner) =
+ Some(tokio::time::Instant::now() + shutdown_timeout);
+ tracing::info!("gateway draining");
+ let _ = draining_tx.send(());
+ })
+ .into_future();
tokio::pin!(server);
state.ready.store(true, Ordering::Release);
let result = tokio::select! {
diff --git a/ai-gateway/src/server/connection.rs b/ai-gateway/src/server/connection.rs
new file mode 100644
index 0000000000..babb32403f
--- /dev/null
+++ b/ai-gateway/src/server/connection.rs
@@ -0,0 +1,189 @@
+//! Socket write accounting for accepted connections, logged when each closes.
+//!
+//! The relay reports a stream complete once hyper has taken the last body chunk,
+//! which says nothing about whether those bytes reached the socket. These counters
+//! do: `bytes_written` is what the kernel send buffer accepted, and a peer that
+//! stops reading shows up as a write still blocked at close once that buffer fills.
+//!
+//! Temporary diagnostics for streams the load balancer holds open until its idle
+//! timeout after the gateway reported them complete. Delete this module, its
+//! `ConnectInfo` wiring in `server::serve`, and the `gateway.connection.id` span
+//! and correlation fields once that cause is known.
+use std::{
+ io,
+ net::SocketAddr,
+ pin::Pin,
+ sync::atomic::{AtomicU64, Ordering},
+ task::{Context, Poll},
+ time::Duration,
+};
+
+use axum::{
+ extract::connect_info::Connected,
+ serve::{IncomingStream, Listener},
+};
+use tokio::{
+ io::{AsyncRead, AsyncWrite, ReadBuf},
+ net::{TcpListener, TcpStream},
+ time::Instant,
+};
+
+/// Connections that wrote less than this without a write error or a long
+/// write block, such as health checks, are not logged.
+const PROBE_BYTES: u64 = 8 * 1024;
+
+/// A new socket's first write is often pending until the reactor sees it
+/// writable, so shorter blocks are not worth a log line on their own.
+const NOTABLE_WRITE_BLOCK: Duration = Duration::from_millis(100);
+
+static NEXT_CONNECTION_ID: AtomicU64 = AtomicU64::new(1);
+
+/// Wraps accepted sockets in [`ObservedStream`].
+pub(crate) struct ObservedListener(pub(crate) TcpListener);
+
+impl Listener for ObservedListener {
+ type Io = ObservedStream;
+ type Addr = SocketAddr;
+
+ async fn accept(&mut self) -> (Self::Io, Self::Addr) {
+ let (io, peer) = Listener::accept(&mut self.0).await;
+ (ObservedStream::new(io, peer), peer)
+ }
+
+ fn local_addr(&self) -> io::Result<Self::Addr> {
+ self.0.local_addr()
+ }
+}
+
+/// Process-unique ID of the connection that carried a request, recorded on the
+/// server span so request logs join the connection's close log.
+#[derive(Clone, Debug)]
+pub(crate) struct ConnectionId(pub(crate) String);
+
+impl Connected<IncomingStream<'_, ObservedListener>> for ConnectionId {
+ fn connect_info(stream: IncomingStream<'_, ObservedListener>) -> Self {
+ Self(stream.io().id.to_string())
+ }
+}
+
+pub(crate) struct ObservedStream {
+ io: TcpStream,
+ id: u64,
+ peer: SocketAddr,
+ opened: Instant,
+ bytes_written: u64,
+ last_write: Option<Instant>,
+ write_blocked_since: Option<Instant>,
+ longest_write_block: Duration,
+ write_error: Option<io::ErrorKind>,
+}
+
+impl ObservedStream {
+ fn new(io: TcpStream, peer: SocketAddr) -> Self {
+ Self {
+ io,
+ id: NEXT_CONNECTION_ID.fetch_add(1, Ordering::Relaxed),
+ peer,
+ opened: Instant::now(),
+ bytes_written: 0,
+ last_write: None,
+ write_blocked_since: None,
+ longest_write_block: Duration::ZERO,
+ write_error: None,
+ }
+ }
+
+ fn record_write(&mut self, poll: Poll<io::Result<usize>>) -> Poll<io::Result<usize>> {
+ let now = Instant::now();
+ match &poll {
+ Poll::Pending => {
+ self.write_blocked_since.get_or_insert(now);
+ return poll;
+ }
+ Poll::Ready(Ok(written)) => {
+ self.bytes_written += *written as u64;
+ self.last_write = Some(now);
+ }
+ Poll::Ready(Err(error)) => {
+ self.write_error.get_or_insert_with(|| error.kind());
+ }
+ }
+ if let Some(since) = self.write_blocked_since.take() {
+ self.longest_write_block = self.longest_write_block.max(now - since);
+ }
+ poll
+ }
+}
+
+impl AsyncRead for ObservedStream {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.get_mut().io).poll_read(cx, buf)
+ }
+}
+
+impl AsyncWrite for ObservedStream {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ buf: &[u8],
+ ) -> Poll<io::Result<usize>> {
+ let this = self.get_mut();
+ let poll = Pin::new(&mut this.io).poll_write(cx, buf);
+ this.record_write(poll)
+ }
+
+ fn poll_write_vectored(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ bufs: &[io::IoSlice<'_>],
+ ) -> Poll<io::Result<usize>> {
+ let this = self.get_mut();
+ let poll = Pin::new(&mut this.io).poll_write_vectored(cx, bufs);
+ this.record_write(poll)
+ }
+
+ fn is_write_vectored(&self) -> bool {
+ self.io.is_write_vectored()
+ }
+
+ fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.get_mut().io).poll_flush(cx)
+ }
+
+ fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+ Pin::new(&mut self.get_mut().io).poll_shutdown(cx)
+ }
+}
+
+impl Drop for ObservedStream {
+ fn drop(&mut self) {
+ let now = Instant::now();
+ let blocked_at_close = self.write_blocked_since.map(|since| now - since);
+ if self.bytes_written < PROBE_BYTES
+ && self.write_error.is_none()
+ && blocked_at_close.is_none_or(|blocked| blocked < NOTABLE_WRITE_BLOCK)
+ && self.longest_write_block < NOTABLE_WRITE_BLOCK
+ {
+ return;
+ }
+ let millis = |duration: Duration| duration.as_secs_f64() * 1000.0;
+ tracing::info!(
+ connection_id = %self.id,
+ peer = %self.peer,
+ duration_ms = millis(now - self.opened),
+ bytes_written = self.bytes_written,
+ since_last_write_ms = self.last_write.map(|last| millis(now - last)),
+ longest_write_block_ms = millis(self.longest_write_block),
+ write_blocked_at_close_ms = blocked_at_close.map(millis),
+ write_error = self.write_error.map(tracing::field::display),
+ "gateway connection closed"
+ );
+ }
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/ai-gateway/src/server/connection/tests.rs b/ai-gateway/src/server/connection/tests.rs
new file mode 100644
index 0000000000..052ca5d411
--- /dev/null
+++ b/ai-gateway/src/server/connection/tests.rs
@@ -0,0 +1,163 @@
+use super::*;
+use crate::{observability, server::GatewayLifecycleState};
+use axum::{Router, routing::get};
+use serde_json::Value;
+use std::{
+ sync::{Arc, Mutex},
+ task::Waker,
+};
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+use tracing_subscriber::prelude::*;
+
+#[derive(Clone, Default)]
+struct Buffer(Arc<Mutex<Vec<u8>>>);
+impl io::Write for Buffer {
+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
+ self.0.lock().unwrap().extend_from_slice(bytes);
+ Ok(bytes.len())
+ }
+ fn flush(&mut self) -> io::Result<()> {
+ Ok(())
+ }
+}
+
+fn record_logs() -> (tracing::subscriber::DefaultGuard, Buffer) {
+ let output = Buffer::default();
+ let buffer = output.clone();
+ let subscriber = tracing_subscriber::registry()
+ .with(
+ tracing_subscriber::fmt::layer()
+ .json()
+ .flatten_event(true)
+ .map_event_format(observability::logs::WithCorrelationFields)
+ .with_writer(move || buffer.clone()),
+ )
+ .with(observability::logs::CorrelationSpans);
+ (tracing::subscriber::set_default(subscriber), output)
+}
+
+fn events(output: &Buffer, message: &str) -> Vec<Value> {
+ String::from_utf8(output.0.lock().unwrap().clone())
+ .unwrap()
+ .lines()
+ .map(|line| serde_json::from_str::<Value>(line).unwrap())
+ .filter(|event| event["message"] == message)
+ .collect()
+}
+
+#[tokio::test]
+async fn close_log_counts_socket_bytes_and_joins_request_logs() {
+ const BODY: usize = 64 * 1024;
+ let (_guard, output) = record_logs();
+ let lifecycle = GatewayLifecycleState::default();
+ let app = observability::instrument(
+ Router::new().route("/stream", get(|| async { "a".repeat(BODY) })),
+ );
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let serving = tokio::spawn(crate::server::serve(
+ listener,
+ app,
+ lifecycle,
+ std::future::pending(),
+ Duration::from_secs(1),
+ ));
+ let mut client = TcpStream::connect(address).await.unwrap();
+ let request = b"GET /stream HTTP/1.1\r\nhost: gateway\r\n\r\n";
+ client.write_all(request).await.unwrap();
+ let mut response = Vec::new();
+ let mut chunk = vec![0; 16 * 1024];
+ loop {
+ let read = client.read(&mut chunk).await.unwrap();
+ assert!(read > 0, "connection closed before the body arrived");
+ response.extend_from_slice(&chunk[..read]);
+ let header_end = response.windows(4).position(|window| window == b"\r\n\r\n");
+ if header_end.is_some_and(|end| response.len() - end - 4 >= BODY) {
+ break;
+ }
+ }
+ let received = response.len();
+ drop(client);
+ let closed = tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ if let Some(event) = events(&output, "gateway connection closed").pop() {
+ return event;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .unwrap();
+ serving.abort();
+
+ assert_eq!(closed["bytes_written"], received);
+ assert!(closed["since_last_write_ms"].as_f64().unwrap() >= 0.0);
+ assert!(
+ closed.get("write_blocked_at_close_ms").is_none(),
+ "{closed}"
+ );
+ assert!(closed.get("write_error").is_none(), "{closed}");
+ let started = events(&output, "gateway response started");
+ assert_eq!(started.len(), 1);
+ assert_eq!(started[0]["connection_id"], closed["connection_id"]);
+}
+
+#[tokio::test]
+async fn write_blocked_by_a_peer_that_stops_reading_is_logged_at_close() {
+ let (_guard, output) = record_logs();
+ let mut listener = ObservedListener(TcpListener::bind("127.0.0.1:0").await.unwrap());
+ let client = TcpStream::connect(listener.local_addr().unwrap())
+ .await
+ .unwrap();
+ let (mut stream, _) = Listener::accept(&mut listener).await;
+ stream.io.writable().await.unwrap();
+ let chunk = vec![0; 64 * 1024];
+ let mut cx = Context::from_waker(Waker::noop());
+ while let Poll::Ready(written) = Pin::new(&mut stream).poll_write(&mut cx, &chunk) {
+ assert!(written.unwrap() > 0);
+ }
+ let written = stream.bytes_written;
+ assert!(written > 0);
+ assert!(stream.write_blocked_since.is_some());
+ drop(stream);
+ drop(client);
+
+ let closed = events(&output, "gateway connection closed");
+ assert_eq!(closed.len(), 1);
+ assert_eq!(closed[0]["bytes_written"], written);
+ assert!(closed[0]["write_blocked_at_close_ms"].as_f64().unwrap() >= 0.0);
+}
+
+#[tokio::test(start_paused = true)]
+async fn small_connections_with_a_recovered_write_block_are_logged() {
+ let (_guard, output) = record_logs();
+ let mut listener = ObservedListener(TcpListener::bind("127.0.0.1:0").await.unwrap());
+ let _client = TcpStream::connect(listener.local_addr().unwrap())
+ .await
+ .unwrap();
+ let (mut stream, _) = Listener::accept(&mut listener).await;
+ let _ = stream.record_write(Poll::Pending);
+ tokio::time::advance(Duration::from_millis(250)).await;
+ let _ = stream.record_write(Poll::Ready(Ok(19)));
+ drop(stream);
+
+ let closed = events(&output, "gateway connection closed");
+ assert_eq!(closed.len(), 1);
+ assert_eq!(closed[0]["bytes_written"], 19);
+ assert_eq!(closed[0]["longest_write_block_ms"], 250.0);
+ assert!(closed[0].get("write_blocked_at_close_ms").is_none());
+}
+
+#[tokio::test]
+async fn probe_connections_are_not_logged() {
+ let (_guard, output) = record_logs();
+ let mut listener = ObservedListener(TcpListener::bind("127.0.0.1:0").await.unwrap());
+ let _client = TcpStream::connect(listener.local_addr().unwrap())
+ .await
+ .unwrap();
+ let (mut stream, _) = Listener::accept(&mut listener).await;
+ stream.write_all(b"HTTP/1.1 200 OK\r\n\r\n").await.unwrap();
+ drop(stream);
+
+ assert!(events(&output, "gateway connection closed").is_empty());
+}