🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
feat(ai-gateway): log per-connection socket writes and output completeness (#19154)
Tobias Wochinger2b8c5c9d37fc
diff --git a/ai-gateway/src/observability/http.rs b/ai-gateway/src/observability/http.rsindex 5f5399952f..774eb84289 100644--- a/ai-gateway/src/observability/http.rs+++ b/ai-gateway/src/observability/http.rs@@ -3,7 +3,7 @@ use std::time::{Duration, Instant};use axum::{Router,body::Body,- extract::MatchedPath,+ extract::{ConnectInfo, MatchedPath},http::{HeaderMap, Request, StatusCode},middleware::{self, Next},response::Response,@@ -14,6 +14,7 @@ use tracing::Span;use tracing_opentelemetry::OpenTelemetrySpanExt;use super::request::{ErrorOrigin, RequestObservation, Summary};+use crate::server::ConnectionId;/// Instrument application routes; mount health probes outside this router.pub fn instrument(router: Router) -> Router {@@ -23,7 +24,10 @@ pub fn instrument(router: Router) -> Router {TraceLayer::new_for_http().make_span_with(|request: &Request<Body>| {let (method, route) = request_labels(request);- let span = tracing::info_span!(parent: None, "http.server", otel.name = %format_args!("{method} {route}"), otel.kind = "server", http.request.method = method, http.route = route, http.request.body.size = tracing::field::Empty, http.response.status_code = tracing::field::Empty, otel.status_code = tracing::field::Empty, otel.status_description = tracing::field::Empty, error.type = tracing::field::Empty, error.origin = tracing::field::Empty, gateway.request.id = tracing::field::Empty, gateway.client.request.id = tracing::field::Empty, langfuse.organization.id = tracing::field::Empty, langfuse.project.id = tracing::field::Empty, gateway.provider = tracing::field::Empty, gateway.provider.connection.id = tracing::field::Empty, gateway.provider.connection.name = tracing::field::Empty, gateway.upstream.model = tracing::field::Empty, provider_request_id = tracing::field::Empty, gateway.outcome = tracing::field::Empty, gateway.first_byte_ms = tracing::field::Empty);+ let span = tracing::info_span!(parent: None, "http.server", otel.name = %format_args!("{method} {route}"), otel.kind = "server", http.request.method = method, http.route = route, http.request.body.size = tracing::field::Empty, http.response.status_code = tracing::field::Empty, otel.status_code = tracing::field::Empty, otel.status_description = tracing::field::Empty, error.type = tracing::field::Empty, error.origin = tracing::field::Empty, gateway.request.id = tracing::field::Empty, gateway.connection.id = tracing::field::Empty, gateway.client.request.id = tracing::field::Empty, langfuse.organization.id = tracing::field::Empty, langfuse.project.id = tracing::field::Empty, gateway.provider = tracing::field::Empty, gateway.provider.connection.id = tracing::field::Empty, gateway.provider.connection.name = tracing::field::Empty, gateway.upstream.model = tracing::field::Empty, provider_request_id = tracing::field::Empty, gateway.outcome = tracing::field::Empty, gateway.first_byte_ms = tracing::field::Empty);+ if let Some(ConnectInfo(ConnectionId(id))) = request.extensions().get::<ConnectInfo<ConnectionId>>() {+ span.record("gateway.connection.id", id.as_str());+ }// Operational tracing has no caller or ambient trace context.let _ = span.set_parent(Context::new());spandiff --git a/ai-gateway/src/observability/http/tests.rs b/ai-gateway/src/observability/http/tests.rsindex f1a774d276..2a511a2e2d 100644--- a/ai-gateway/src/observability/http/tests.rs+++ b/ai-gateway/src/observability/http/tests.rs@@ -714,6 +714,10 @@ fn assert_request_correlation(assert_eq!(event[key], expected, "{event}");}}+ if event["message"] == "gateway execution finished" {+ assert_eq!(event["outcome"], "complete", "{event}");+ assert_eq!(event["output_complete"], true, "{event}");+ }}for (name, headers, payload) in observed {assert_eq!(diff --git a/ai-gateway/src/observability/logs.rs b/ai-gateway/src/observability/logs.rsindex 8097079ba8..8df0a2d339 100644--- a/ai-gateway/src/observability/logs.rs+++ b/ai-gateway/src/observability/logs.rs@@ -15,8 +15,9 @@ use tracing_subscriber::{};/// Span field and its top-level log key. All are recorded on the server span.-const CORRELATION_FIELDS: [(&str, &str); 7] = [+const CORRELATION_FIELDS: [(&str, &str); 8] = [("gateway.request.id", "request_id"),+ ("gateway.connection.id", "connection_id"),("langfuse.organization.id", "organization_id"),("langfuse.project.id", "project_id"),("gateway.provider", "provider"),diff --git a/ai-gateway/src/observability/metrics.rs b/ai-gateway/src/observability/metrics.rsindex fad50b911c..b7e4df3d9f 100644--- a/ai-gateway/src/observability/metrics.rs+++ b/ai-gateway/src/observability/metrics.rs@@ -276,7 +276,7 @@ pub(crate) fn execution_finished(facts: &InferenceFacts) {let context = tracing::Span::current().context();let context_span = context.span();let ids = context_span.span_context();- tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), outcome, duration_ms = facts.duration_ms, "gateway execution finished");+ tracing::info!(trace_id = %ids.trace_id(), span_id = %ids.span_id(), outcome, duration_ms = facts.duration_ms, output_complete = facts.inference.output_complete, "gateway execution finished");}/// Marks the server span failed when the provider body breaks after headers werediff --git a/ai-gateway/src/observability/mod.rs b/ai-gateway/src/observability/mod.rsindex 53936a7f64..8a35665add 100644--- a/ai-gateway/src/observability/mod.rs+++ b/ai-gateway/src/observability/mod.rs@@ -1,6 +1,6 @@mod client;mod http;-mod logs;+pub(crate) mod logs;mod metrics;mod request;diff --git a/ai-gateway/src/server.rs b/ai-gateway/src/server.rsindex 616f59cc8e..ab7aa6c61f 100644--- a/ai-gateway/src/server.rs+++ b/ai-gateway/src/server.rs@@ -12,6 +12,10 @@ use axum::{Json, Router, extract::State, http::StatusCode, routing::get};use serde::Serialize;use tokio::{net::TcpListener, sync::oneshot};+mod connection;++pub(crate) use connection::ConnectionId;+#[derive(Clone, Default)]pub struct GatewayLifecycleState {ready: Arc<AtomicBool>,@@ -85,19 +89,22 @@ pub async fn serve() -> io::Result<()> {let (draining_tx, draining_rx) = oneshot::channel();let shutdown_state = state.clone();- let server = axum::serve(listener, app)- .with_graceful_shutdown(async move {- shutdown.await;- shutdown_state.ready.store(false, Ordering::Release);- *shutdown_state- .drain_deadline- .lock()- .unwrap_or_else(PoisonError::into_inner) =- Some(tokio::time::Instant::now() + shutdown_timeout);- tracing::info!("gateway draining");- let _ = draining_tx.send(());- })- .into_future();+ let server = axum::serve(+ connection::ObservedListener(listener),+ app.into_make_service_with_connect_info::<ConnectionId>(),+ )+ .with_graceful_shutdown(async move {+ shutdown.await;+ shutdown_state.ready.store(false, Ordering::Release);+ *shutdown_state+ .drain_deadline+ .lock()+ .unwrap_or_else(PoisonError::into_inner) =+ Some(tokio::time::Instant::now() + shutdown_timeout);+ tracing::info!("gateway draining");+ let _ = draining_tx.send(());+ })+ .into_future();tokio::pin!(server);state.ready.store(true, Ordering::Release);let result = tokio::select! {diff --git a/ai-gateway/src/server/connection.rs b/ai-gateway/src/server/connection.rsnew file mode 100644index 0000000000..babb32403f--- /dev/null+++ b/ai-gateway/src/server/connection.rs@@ -0,0 +1,189 @@+//! Socket write accounting for accepted connections, logged when each closes.+//!+//! The relay reports a stream complete once hyper has taken the last body chunk,+//! which says nothing about whether those bytes reached the socket. These counters+//! do: `bytes_written` is what the kernel send buffer accepted, and a peer that+//! stops reading shows up as a write still blocked at close once that buffer fills.+//!+//! Temporary diagnostics for streams the load balancer holds open until its idle+//! timeout after the gateway reported them complete. Delete this module, its+//! `ConnectInfo` wiring in `server::serve`, and the `gateway.connection.id` span+//! and correlation fields once that cause is known.+use std::{+ io,+ net::SocketAddr,+ pin::Pin,+ sync::atomic::{AtomicU64, Ordering},+ task::{Context, Poll},+ time::Duration,+};++use axum::{+ extract::connect_info::Connected,+ serve::{IncomingStream, Listener},+};+use tokio::{+ io::{AsyncRead, AsyncWrite, ReadBuf},+ net::{TcpListener, TcpStream},+ time::Instant,+};++/// Connections that wrote less than this without a write error or a long+/// write block, such as health checks, are not logged.+const PROBE_BYTES: u64 = 8 * 1024;++/// A new socket's first write is often pending until the reactor sees it+/// writable, so shorter blocks are not worth a log line on their own.+const NOTABLE_WRITE_BLOCK: Duration = Duration::from_millis(100);++static NEXT_CONNECTION_ID: AtomicU64 = AtomicU64::new(1);++/// Wraps accepted sockets in [`ObservedStream`].+pub(crate) struct ObservedListener(pub(crate) TcpListener);++impl Listener for ObservedListener {+ type Io = ObservedStream;+ type Addr = SocketAddr;++ async fn accept(&mut self) -> (Self::Io, Self::Addr) {+ let (io, peer) = Listener::accept(&mut self.0).await;+ (ObservedStream::new(io, peer), peer)+ }++ fn local_addr(&self) -> io::Result<Self::Addr> {+ self.0.local_addr()+ }+}++/// Process-unique ID of the connection that carried a request, recorded on the+/// server span so request logs join the connection's close log.+#[derive(Clone, Debug)]+pub(crate) struct ConnectionId(pub(crate) String);++impl Connected<IncomingStream<'_, ObservedListener>> for ConnectionId {+ fn connect_info(stream: IncomingStream<'_, ObservedListener>) -> Self {+ Self(stream.io().id.to_string())+ }+}++pub(crate) struct ObservedStream {+ io: TcpStream,+ id: u64,+ peer: SocketAddr,+ opened: Instant,+ bytes_written: u64,+ last_write: Option<Instant>,+ write_blocked_since: Option<Instant>,+ longest_write_block: Duration,+ write_error: Option<io::ErrorKind>,+}++impl ObservedStream {+ fn new(io: TcpStream, peer: SocketAddr) -> Self {+ Self {+ io,+ id: NEXT_CONNECTION_ID.fetch_add(1, Ordering::Relaxed),+ peer,+ opened: Instant::now(),+ bytes_written: 0,+ last_write: None,+ write_blocked_since: None,+ longest_write_block: Duration::ZERO,+ write_error: None,+ }+ }++ fn record_write(&mut self, poll: Poll<io::Result<usize>>) -> Poll<io::Result<usize>> {+ let now = Instant::now();+ match &poll {+ Poll::Pending => {+ self.write_blocked_since.get_or_insert(now);+ return poll;+ }+ Poll::Ready(Ok(written)) => {+ self.bytes_written += *written as u64;+ self.last_write = Some(now);+ }+ Poll::Ready(Err(error)) => {+ self.write_error.get_or_insert_with(|| error.kind());+ }+ }+ if let Some(since) = self.write_blocked_since.take() {+ self.longest_write_block = self.longest_write_block.max(now - since);+ }+ poll+ }+}++impl AsyncRead for ObservedStream {+ fn poll_read(+ self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ buf: &mut ReadBuf<'_>,+ ) -> Poll<io::Result<()>> {+ Pin::new(&mut self.get_mut().io).poll_read(cx, buf)+ }+}++impl AsyncWrite for ObservedStream {+ fn poll_write(+ self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ buf: &[u8],+ ) -> Poll<io::Result<usize>> {+ let this = self.get_mut();+ let poll = Pin::new(&mut this.io).poll_write(cx, buf);+ this.record_write(poll)+ }++ fn poll_write_vectored(+ self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ bufs: &[io::IoSlice<'_>],+ ) -> Poll<io::Result<usize>> {+ let this = self.get_mut();+ let poll = Pin::new(&mut this.io).poll_write_vectored(cx, bufs);+ this.record_write(poll)+ }++ fn is_write_vectored(&self) -> bool {+ self.io.is_write_vectored()+ }++ fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {+ Pin::new(&mut self.get_mut().io).poll_flush(cx)+ }++ fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {+ Pin::new(&mut self.get_mut().io).poll_shutdown(cx)+ }+}++impl Drop for ObservedStream {+ fn drop(&mut self) {+ let now = Instant::now();+ let blocked_at_close = self.write_blocked_since.map(|since| now - since);+ if self.bytes_written < PROBE_BYTES+ && self.write_error.is_none()+ && blocked_at_close.is_none_or(|blocked| blocked < NOTABLE_WRITE_BLOCK)+ && self.longest_write_block < NOTABLE_WRITE_BLOCK+ {+ return;+ }+ let millis = |duration: Duration| duration.as_secs_f64() * 1000.0;+ tracing::info!(+ connection_id = %self.id,+ peer = %self.peer,+ duration_ms = millis(now - self.opened),+ bytes_written = self.bytes_written,+ since_last_write_ms = self.last_write.map(|last| millis(now - last)),+ longest_write_block_ms = millis(self.longest_write_block),+ write_blocked_at_close_ms = blocked_at_close.map(millis),+ write_error = self.write_error.map(tracing::field::display),+ "gateway connection closed"+ );+ }+}++#[cfg(test)]+mod tests;diff --git a/ai-gateway/src/server/connection/tests.rs b/ai-gateway/src/server/connection/tests.rsnew file mode 100644index 0000000000..052ca5d411--- /dev/null+++ b/ai-gateway/src/server/connection/tests.rs@@ -0,0 +1,163 @@+use super::*;+use crate::{observability, server::GatewayLifecycleState};+use axum::{Router, routing::get};+use serde_json::Value;+use std::{+ sync::{Arc, Mutex},+ task::Waker,+};+use tokio::io::{AsyncReadExt, AsyncWriteExt};+use tracing_subscriber::prelude::*;++#[derive(Clone, Default)]+struct Buffer(Arc<Mutex<Vec<u8>>>);+impl io::Write for Buffer {+ fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {+ self.0.lock().unwrap().extend_from_slice(bytes);+ Ok(bytes.len())+ }+ fn flush(&mut self) -> io::Result<()> {+ Ok(())+ }+}++fn record_logs() -> (tracing::subscriber::DefaultGuard, Buffer) {+ let output = Buffer::default();+ let buffer = output.clone();+ let subscriber = tracing_subscriber::registry()+ .with(+ tracing_subscriber::fmt::layer()+ .json()+ .flatten_event(true)+ .map_event_format(observability::logs::WithCorrelationFields)+ .with_writer(move || buffer.clone()),+ )+ .with(observability::logs::CorrelationSpans);+ (tracing::subscriber::set_default(subscriber), output)+}++fn events(output: &Buffer, message: &str) -> Vec<Value> {+ String::from_utf8(output.0.lock().unwrap().clone())+ .unwrap()+ .lines()+ .map(|line| serde_json::from_str::<Value>(line).unwrap())+ .filter(|event| event["message"] == message)+ .collect()+}++#[tokio::test]+async fn close_log_counts_socket_bytes_and_joins_request_logs() {+ const BODY: usize = 64 * 1024;+ let (_guard, output) = record_logs();+ let lifecycle = GatewayLifecycleState::default();+ let app = observability::instrument(+ Router::new().route("/stream", get(|| async { "a".repeat(BODY) })),+ );+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let serving = tokio::spawn(crate::server::serve(+ listener,+ app,+ lifecycle,+ std::future::pending(),+ Duration::from_secs(1),+ ));+ let mut client = TcpStream::connect(address).await.unwrap();+ let request = b"GET /stream HTTP/1.1\r\nhost: gateway\r\n\r\n";+ client.write_all(request).await.unwrap();+ let mut response = Vec::new();+ let mut chunk = vec![0; 16 * 1024];+ loop {+ let read = client.read(&mut chunk).await.unwrap();+ assert!(read > 0, "connection closed before the body arrived");+ response.extend_from_slice(&chunk[..read]);+ let header_end = response.windows(4).position(|window| window == b"\r\n\r\n");+ if header_end.is_some_and(|end| response.len() - end - 4 >= BODY) {+ break;+ }+ }+ let received = response.len();+ drop(client);+ let closed = tokio::time::timeout(Duration::from_secs(5), async {+ loop {+ if let Some(event) = events(&output, "gateway connection closed").pop() {+ return event;+ }+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ })+ .await+ .unwrap();+ serving.abort();++ assert_eq!(closed["bytes_written"], received);+ assert!(closed["since_last_write_ms"].as_f64().unwrap() >= 0.0);+ assert!(+ closed.get("write_blocked_at_close_ms").is_none(),+ "{closed}"+ );+ assert!(closed.get("write_error").is_none(), "{closed}");+ let started = events(&output, "gateway response started");+ assert_eq!(started.len(), 1);+ assert_eq!(started[0]["connection_id"], closed["connection_id"]);+}++#[tokio::test]+async fn write_blocked_by_a_peer_that_stops_reading_is_logged_at_close() {+ let (_guard, output) = record_logs();+ let mut listener = ObservedListener(TcpListener::bind("127.0.0.1:0").await.unwrap());+ let client = TcpStream::connect(listener.local_addr().unwrap())+ .await+ .unwrap();+ let (mut stream, _) = Listener::accept(&mut listener).await;+ stream.io.writable().await.unwrap();+ let chunk = vec![0; 64 * 1024];+ let mut cx = Context::from_waker(Waker::noop());+ while let Poll::Ready(written) = Pin::new(&mut stream).poll_write(&mut cx, &chunk) {+ assert!(written.unwrap() > 0);+ }+ let written = stream.bytes_written;+ assert!(written > 0);+ assert!(stream.write_blocked_since.is_some());+ drop(stream);+ drop(client);++ let closed = events(&output, "gateway connection closed");+ assert_eq!(closed.len(), 1);+ assert_eq!(closed[0]["bytes_written"], written);+ assert!(closed[0]["write_blocked_at_close_ms"].as_f64().unwrap() >= 0.0);+}++#[tokio::test(start_paused = true)]+async fn small_connections_with_a_recovered_write_block_are_logged() {+ let (_guard, output) = record_logs();+ let mut listener = ObservedListener(TcpListener::bind("127.0.0.1:0").await.unwrap());+ let _client = TcpStream::connect(listener.local_addr().unwrap())+ .await+ .unwrap();+ let (mut stream, _) = Listener::accept(&mut listener).await;+ let _ = stream.record_write(Poll::Pending);+ tokio::time::advance(Duration::from_millis(250)).await;+ let _ = stream.record_write(Poll::Ready(Ok(19)));+ drop(stream);++ let closed = events(&output, "gateway connection closed");+ assert_eq!(closed.len(), 1);+ assert_eq!(closed[0]["bytes_written"], 19);+ assert_eq!(closed[0]["longest_write_block_ms"], 250.0);+ assert!(closed[0].get("write_blocked_at_close_ms").is_none());+}++#[tokio::test]+async fn probe_connections_are_not_logged() {+ let (_guard, output) = record_logs();+ let mut listener = ObservedListener(TcpListener::bind("127.0.0.1:0").await.unwrap());+ let _client = TcpStream::connect(listener.local_addr().unwrap())+ .await+ .unwrap();+ let (mut stream, _) = Listener::accept(&mut listener).await;+ stream.write_all(b"HTTP/1.1 200 OK\r\n\r\n").await.unwrap();+ drop(stream);++ assert!(events(&output, "gateway connection closed").is_empty());+}