:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Tower-layer instrumentation for inbound POST body buffering.
//!
//! axum's `Bytes` extractor reads and buffers the entire request body
//! before the handler observes it, but that buffering happens *inside*
//! the handler's invocation — so on a slow client upload, the latency
//! is charged to the handler with no per-step visibility. This shim
//! buffers the body explicitly inside a middleware layer placed
//! immediately after [`crate::api::concurrency_metrics::record_concurrency_wait`]
//! and before the handler. Side effects:
//!
//! 1. The elapsed wall-clock duration of `axum::body::to_bytes` is
//! stamped onto the request extensions as [`BodyReadDuration`].
//! 2. The original body is replaced with `Body::from(bytes)`, so the
//! handler's `body: Bytes` extractor reads from in-memory and the
//! overall behavior is unchanged.
//!
//! Layer wiring lives in `router::router` — see the comment block there.
//! The shim no-ops gracefully on the `Bytes` extractor's downstream
//! contract: if buffering fails, the framework's existing 400 response
//! shape is preserved.
//!
//! # Why no team_id label
//!
//! Body buffering happens before authentication resolves a team. The
//! histogram is pod-level by design, mirroring
//! `flags_concurrency_limit_wait_ms`.
use std::time::{Duration, Instant};
use axum::{
body::Body,
extract::Request,
http::StatusCode,
middleware::Next,
response::{IntoResponse, Response},
};
use common_metrics::inc;
use crate::metrics::consts::{FLAG_BODY_READ_FAILED_COUNTER, FLAG_BODY_READ_TOO_LARGE_COUNTER};
/// Maximum inbound POST body size for `/flags` and `/decide`. Kept in
/// lockstep with the `DefaultBodyLimit` layer on the same sub-router so
/// the streaming cap (here) and the extractor cap (downstream) reject at
/// the same boundary.
pub const MAX_FLAGS_BODY_BYTES: usize = 2 * 1024 * 1024;
/// Wall-clock duration spent buffering the inbound request body. Inserted
/// into request extensions by [`record_body_read`] and read by the
/// `flags` handler to populate `FlagsCanonicalLogLine::body_read_ms`.
#[derive(Clone, Copy, Debug)]
pub struct BodyReadDuration(pub Duration);
/// Buffers the request body to memory while timing the operation, then
/// forwards a request with an in-memory body so the handler's `Bytes`
/// extractor short-circuits.
///
/// Bodies above [`MAX_FLAGS_BODY_BYTES`] are rejected with 413; transport
/// errors are rejected with 400. Both responses use the same body string
/// that axum's `BytesRejection::FailedToBufferBody` would produce.
pub async fn record_body_read(req: Request, next: Next) -> Response {
let (parts, body) = req.into_parts();
let start = Instant::now();
let bytes = match axum::body::to_bytes(body, MAX_FLAGS_BODY_BYTES).await {
Ok(bytes) => bytes,
Err(err) => {
// `to_bytes` wraps the underlying http_body_util error. Walk
// the full `Error::source` chain rather than just the first
// level: axum currently wraps `LengthLimitError` one layer
// deep, but that depth is internal to axum/http-body-util,
// not a contract. A future bump that adds another wrapper
// layer would silently misclassify 413s as 400s and stop
// firing `flags_body_read_too_large_total`.
let too_large = std::iter::successors(std::error::Error::source(&err), |s| s.source())
.any(|s| s.is::<http_body_util::LengthLimitError>());
if too_large {
inc(FLAG_BODY_READ_TOO_LARGE_COUNTER, &[], 1);
tracing::warn!(
error = %err,
limit = MAX_FLAGS_BODY_BYTES,
"request body exceeded limit"
);
return (
StatusCode::PAYLOAD_TOO_LARGE,
"Failed to buffer the request body",
)
.into_response();
}
inc(FLAG_BODY_READ_FAILED_COUNTER, &[], 1);
tracing::warn!(error = %err, "failed to buffer request body");
return (StatusCode::BAD_REQUEST, "Failed to buffer the request body").into_response();
}
};
let elapsed = start.elapsed();
let mut req = Request::from_parts(parts, Body::from(bytes));
req.extensions_mut().insert(BodyReadDuration(elapsed));
next.run(req).await
}
#[cfg(test)]
mod tests {
//! These tests exercise the shim's contract directly — buffer the
//! body, time the operation, hand off a request with an in-memory
//! body — without depending on the rest of the layer chain. The
//! placement after `record_concurrency_wait` is enforced by the
//! router wiring; we test the shim itself here.
use super::*;
use axum::{
body::Body,
extract::Extension,
http::{Request as HttpRequest, StatusCode},
routing::post,
Router,
};
use bytes::Bytes;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use tower::ServiceExt;
/// Encodes responses as `<wait_micros|"missing">|<body_bytes>` so
/// tests can split on the first `|` and compare body bytes verbatim.
const ECHO_DELIM: u8 = b'|';
async fn echo_body_read(
wait: Option<Extension<BodyReadDuration>>,
body: Bytes,
) -> (StatusCode, Vec<u8>) {
let wait_repr = wait
.map(|Extension(w)| w.0.as_micros().to_string())
.unwrap_or_else(|| "missing".to_string());
let mut out = wait_repr.into_bytes();
out.push(ECHO_DELIM);
out.extend_from_slice(&body);
(StatusCode::OK, out)
}
fn make_request(body: &str) -> HttpRequest<Body> {
HttpRequest::builder()
.method("POST")
.uri("/")
.body(Body::from(body.to_owned()))
.unwrap()
}
async fn read_response_bytes(resp: Response) -> Vec<u8> {
axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap()
.to_vec()
}
fn split_echo(payload: &[u8]) -> (&str, &[u8]) {
let i = payload
.iter()
.position(|b| *b == ECHO_DELIM)
.expect("echo response must contain the delimiter");
let wait =
std::str::from_utf8(&payload[..i]).expect("wait segment must be ASCII micros/marker");
(wait, &payload[i + 1..])
}
#[tokio::test]
async fn shim_records_duration_and_preserves_body_bytes() {
// Mix of NUL, invalid UTF-8, and high-bit bytes — picked so any
// accidental string conversion or truncation surfaces.
let body: Vec<u8> = vec![
0x00, 0xFF, 0xC3, 0x28, b'h', b'e', b'l', b'l', b'o', 0x80, 0x81, 0x82,
];
let app = Router::new()
.route("/", post(echo_body_read))
.layer(axum::middleware::from_fn(record_body_read));
let req = HttpRequest::builder()
.method("POST")
.uri("/")
.body(Body::from(body.clone()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let payload = read_response_bytes(resp).await;
let (wait, echoed_body) = split_echo(&payload);
assert_ne!(wait, "missing");
assert_eq!(echoed_body, body);
}
#[tokio::test]
async fn shim_handles_empty_body() {
// POST with an empty body still produces a (~0 ms) duration and
// forwards without error.
let app = Router::new()
.route("/", post(echo_body_read))
.layer(axum::middleware::from_fn(record_body_read));
let req = HttpRequest::builder()
.method("POST")
.uri("/")
.body(Body::empty())
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let payload = read_response_bytes(resp).await;
let (wait, echoed_body) = split_echo(&payload);
assert_ne!(wait, "missing");
assert!(echoed_body.is_empty());
}
#[tokio::test]
async fn shim_extension_absent_when_layer_omitted() {
let app = Router::new().route("/", post(echo_body_read));
let resp = app.oneshot(make_request("payload")).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let payload = read_response_bytes(resp).await;
let (wait, echoed_body) = split_echo(&payload);
assert_eq!(wait, "missing");
assert_eq!(echoed_body, b"payload");
}
#[tokio::test]
async fn shim_returns_400_and_increments_failed_counter_on_buffering_error() {
use futures::stream;
// `set_default_local_recorder` returns a thread-local guard;
// safe across `.await` because `#[tokio::test]` defaults to a
// current-thread runtime.
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let _guard = metrics::set_default_local_recorder(&recorder);
let app = Router::new()
.route("/", post(echo_body_read))
.layer(axum::middleware::from_fn(record_body_read));
let err_stream = stream::once(async {
Err::<Bytes, std::io::Error>(std::io::Error::other("simulated upload failure"))
});
let req = HttpRequest::builder()
.method("POST")
.uri("/")
.body(Body::from_stream(err_stream))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
// Pin the shim's own 400 message so we know the response did
// not come from a downstream that observed an empty `Bytes`.
let body_bytes = read_response_bytes(resp).await;
assert_eq!(body_bytes, b"Failed to buffer the request body");
let snapshot = snapshotter.snapshot().into_vec();
let failed = snapshot.iter().find_map(|(ckey, _, _, value)| {
(ckey.key().name() == FLAG_BODY_READ_FAILED_COUNTER).then_some(value)
});
match failed {
Some(DebugValue::Counter(n)) => assert_eq!(*n, 1),
other => {
panic!("expected `{FLAG_BODY_READ_FAILED_COUNTER}` Counter sample, got {other:?}")
}
}
}
#[tokio::test]
async fn shim_accepts_body_at_limit() {
let body = vec![b'a'; MAX_FLAGS_BODY_BYTES];
let app = Router::new()
.route("/", post(echo_body_read))
.layer(axum::middleware::from_fn(record_body_read));
let req = HttpRequest::builder()
.method("POST")
.uri("/")
.body(Body::from(body.clone()))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let payload = read_response_bytes(resp).await;
let (wait, echoed_body) = split_echo(&payload);
assert_ne!(wait, "missing");
assert_eq!(echoed_body, body);
}
#[tokio::test]
async fn shim_returns_413_and_increments_too_large_counter_when_body_exceeds_limit() {
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
let _guard = metrics::set_default_local_recorder(&recorder);
let app = Router::new()
.route("/", post(echo_body_read))
.layer(axum::middleware::from_fn(record_body_read));
let body = vec![b'a'; MAX_FLAGS_BODY_BYTES + 1];
let req = HttpRequest::builder()
.method("POST")
.uri("/")
.body(Body::from(body))
.unwrap();
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE);
let body_bytes = read_response_bytes(resp).await;
assert_eq!(body_bytes, b"Failed to buffer the request body");
let snapshot = snapshotter.snapshot().into_vec();
let too_large = snapshot.iter().find_map(|(ckey, _, _, value)| {
(ckey.key().name() == FLAG_BODY_READ_TOO_LARGE_COUNTER).then_some(value)
});
match too_large {
Some(DebugValue::Counter(n)) => assert_eq!(*n, 1),
other => panic!(
"expected `{FLAG_BODY_READ_TOO_LARGE_COUNTER}` Counter sample, got {other:?}"
),
}
// The transport-failure counter must stay clean — the two paths
// are split on purpose, so a regression that collapses them back
// would cross-fire here.
let failed = snapshot.iter().find_map(|(ckey, _, _, value)| {
(ckey.key().name() == FLAG_BODY_READ_FAILED_COUNTER).then_some(value)
});
if let Some(DebugValue::Counter(n)) = failed {
assert_eq!(
*n, 0,
"transport-failure counter must not fire on size-exceeded path"
);
}
}
}