Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.8 KB
Raw Blame
use std::time::Duration;

use async_compression::tokio::bufread;
use axum::body::Body;
use bytes::{BufMut, Bytes, BytesMut};
use futures::StreamExt;
use tokio::io::AsyncReadExt;

use crate::extractors::{drain_rejected_body, REJECTED_BODY_DRAIN_DEADLINE};
use crate::v1::constants::{CAPTURE_V1_BODY_READ_TIMEOUT, CAPTURE_V1_DECOMPRESSION_ERRORS};
use crate::v1::Error;

/// Map a validated Content-Encoding value to a static metric label value.
///
/// Mirrors `SUPPORTED_ENCODINGS` (gzip, deflate, br, zstd). `None` means no
/// Content-Encoding header was sent ("identity"). `Context::new` already
/// rejects unsupported encodings, so the "other" fallback is a defensive
/// branch that is unreachable in practice.
pub fn encoding_tag(enc: Option<&str>) -> &'static str {
    match enc {
        None => "identity",
        Some("gzip") => "gzip",
        Some("deflate") => "deflate",
        Some("br") => "br",
        Some("zstd") => "zstd",
        Some(_) => "other",
    }
}

/// Extract body bytes from a streaming Body with a per-chunk timeout.
///
/// If `chunk_timeout` is None, reads the entire body without timeout.
/// If `chunk_timeout` is Some, each chunk read is wrapped in a timeout. If no data
/// arrives within the timeout window, returns `Error::BodyReadTimeout`.
///
/// The `payload_size_limit` parameter enforces a maximum body size during streaming.
pub async fn extract_body_with_timeout(
    body: Body,
    payload_size_limit: usize,
    chunk_timeout: Option<Duration>,
    chunk_size_kb: usize,
    path: &'static str,
) -> Result<Bytes, Error> {
    let mut stream = body.into_data_stream();
    let mut buf = BytesMut::with_capacity(std::cmp::min(payload_size_limit, chunk_size_kb * 1024));

    loop {
        let chunk_result = match chunk_timeout {
            Some(timeout) => match tokio::time::timeout(timeout, stream.next()).await {
                Ok(result) => result,
                Err(_elapsed) => {
                    metrics::counter!(CAPTURE_V1_BODY_READ_TIMEOUT, "path" => path).increment(1);
                    return Err(Error::BodyReadTimeout(buf.len()));
                }
            },
            None => stream.next().await,
        };

        match chunk_result {
            Some(Ok(chunk)) => {
                if buf.len() + chunk.len() > payload_size_limit {
                    // The drain can run for seconds. Holding the buffer and the
                    // oversize chunk across it turns every rejection into a memory
                    // spike the size of the limit we just refused.
                    drop(buf);
                    drop(chunk);
                    // Consume what is left so the 413 reaches the client rather than a
                    // connection reset. Bounded by the size we were willing to accept
                    // in the first place.
                    drain_rejected_body(
                        &mut stream,
                        payload_size_limit,
                        REJECTED_BODY_DRAIN_DEADLINE,
                        path,
                    )
                    .await;
                    return Err(Error::PayloadTooLarge(format!(
                        "Request body exceeds limit of {payload_size_limit} bytes"
                    )));
                }
                buf.put(chunk);
            }
            Some(Err(e)) => {
                // A LengthLimitError reaches us only when something wraps the body
                // in http_body_util::Limited. Nothing does on this route today:
                // the v1 DefaultBodyLimit layer only inserts a request extension,
                // and the `Body` extractor never reads it. Kept so the status stays
                // correct if a limited body is ever introduced upstream.
                if error_chain_has_length_limit(&e) {
                    return Err(Error::PayloadTooLarge(format!(
                        "Request body exceeds limit of {payload_size_limit} bytes"
                    )));
                }
                return Err(Error::RequestDecodingError(format!(
                    "Error reading request body: {e:#}"
                )));
            }
            None => break,
        }
    }

    let bytes = buf.freeze();
    if bytes.is_empty() {
        return Err(Error::EmptyBody);
    }
    Ok(bytes)
}

fn error_chain_has_length_limit(err: &axum::Error) -> bool {
    let mut source = std::error::Error::source(err);
    while let Some(e) = source {
        if e.is::<http_body_util::LengthLimitError>() {
            return true;
        }
        source = e.source();
    }
    false
}

/// Decompress a payload using the specified Content-Encoding.
///
/// Returns bytes unchanged when `content_encoding` is `None`. For recognized
/// encodings (gzip, deflate, br, zstd), streams through the appropriate
/// `async-compression` decoder with a size limit enforced per-chunk.
pub async fn decompress_payload(
    content_encoding: Option<&str>,
    bytes: Bytes,
    payload_size_limit: usize,
    chunk_size_kb: usize,
) -> Result<Bytes, Error> {
    let encoding = match content_encoding {
        Some(enc) => enc,
        None => return Ok(bytes),
    };

    let first_byte = bytes.first().copied();
    let reader = tokio::io::BufReader::new(std::io::Cursor::new(bytes));

    match encoding {
        "gzip" => {
            read_decompressed(
                bufread::GzipDecoder::new(reader),
                payload_size_limit,
                chunk_size_kb,
                encoding,
            )
            .await
        }
        // RFC 9110: Content-Encoding "deflate" = zlib (RFC 1950, 0x78 prefix).
        // Sniff first byte: zlib-wrapped → ZlibDecoder, raw → DeflateDecoder.
        "deflate" if first_byte == Some(0x78) => {
            read_decompressed(
                bufread::ZlibDecoder::new(reader),
                payload_size_limit,
                chunk_size_kb,
                encoding,
            )
            .await
        }
        "deflate" => {
            read_decompressed(
                bufread::DeflateDecoder::new(reader),
                payload_size_limit,
                chunk_size_kb,
                encoding,
            )
            .await
        }
        "br" => {
            read_decompressed(
                bufread::BrotliDecoder::new(reader),
                payload_size_limit,
                chunk_size_kb,
                encoding,
            )
            .await
        }
        "zstd" => {
            read_decompressed(
                bufread::ZstdDecoder::new(reader),
                payload_size_limit,
                chunk_size_kb,
                encoding,
            )
            .await
        }
        other => Err(Error::UnsupportedEncoding(other.to_string())),
    }
}

async fn read_decompressed(
    mut reader: impl tokio::io::AsyncRead + Unpin,
    limit: usize,
    chunk_size_kb: usize,
    encoding: &str,
) -> Result<Bytes, Error> {
    let chunk_size = chunk_size_kb * 1024;
    let mut buf = BytesMut::with_capacity(std::cmp::min(limit, chunk_size));
    loop {
        if buf.len() >= limit {
            return Err(Error::PayloadTooLarge(format!(
                "decompressed {encoding} payload exceeds limit of {limit} bytes"
            )));
        }
        let remaining = limit - buf.len();
        let to_read = std::cmp::min(remaining, chunk_size);
        buf.reserve(to_read);
        let n = reader.read_buf(&mut buf).await.map_err(|e| {
            metrics::counter!(CAPTURE_V1_DECOMPRESSION_ERRORS, "encoding" => encoding_tag(Some(encoding)))
                .increment(1);
            Error::RequestDecodingError(format!("{encoding} decompression failed: {e:#}"))
        })?;
        if n == 0 {
            break;
        }
    }
    Ok(buf.freeze())
}

#[cfg(test)]
mod tests {
    use super::*;
    use axum::body::Body;
    use bytes::Bytes;
    use flate2::write::{DeflateEncoder, GzEncoder, ZlibEncoder};
    use flate2::Compression;
    use futures::stream;
    use std::io::Write;
    use std::time::Duration;

    const TEST_CHUNK_SIZE_KB: usize = 256;

    #[test]
    fn encoding_tag_covers_supported_encodings_and_identity() {
        // Keep in lockstep with SUPPORTED_ENCODINGS.
        for enc in crate::v1::constants::SUPPORTED_ENCODINGS {
            assert_eq!(
                encoding_tag(Some(enc)),
                *enc,
                "supported encoding {enc} must map to itself"
            );
        }
        assert_eq!(encoding_tag(None), "identity");
        assert_eq!(encoding_tag(Some("lz4")), "other");
    }

    #[tokio::test]
    async fn extract_body_no_timeout() {
        let body = Body::from("hello world");
        let result = extract_body_with_timeout(body, 1024, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(result.is_ok());
        assert_eq!(result.unwrap(), Bytes::from("hello world"));
    }

    #[tokio::test]
    async fn extract_body_with_timeout_success() {
        let body = Body::from("hello world");
        let timeout = Some(Duration::from_secs(5));
        let result =
            extract_body_with_timeout(body, 1024, timeout, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(result.is_ok());
        assert_eq!(result.unwrap(), Bytes::from("hello world"));
    }

    #[tokio::test]
    async fn extract_body_exceeds_limit() {
        let body = Body::from("hello world this is a long message");
        let result = extract_body_with_timeout(body, 10, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn extract_body_drains_an_oversize_body_before_rejecting() {
        use crate::extractors::test_support::counted_chunks;
        use std::sync::atomic::Ordering;

        // Four ten-byte chunks against a 25-byte limit: the third trips it, and the
        // fourth must still be pulled so hyper can deliver the 413 on a live
        // connection instead of resetting.
        let (stream, pulled) = counted_chunks(4);
        let body = Body::from_stream(stream);

        let result = extract_body_with_timeout(body, 25, None, TEST_CHUNK_SIZE_KB, "/test").await;

        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
        assert_eq!(pulled.load(Ordering::SeqCst), 4);
    }

    #[tokio::test]
    async fn extract_body_timeout_fires() {
        let chunks: Vec<Result<Bytes, std::io::Error>> = vec![Ok(Bytes::from("partial"))];
        let slow_stream = stream::iter(chunks).chain(stream::pending());
        let body = Body::from_stream(slow_stream);

        let timeout = Some(Duration::from_millis(50));
        let result =
            extract_body_with_timeout(body, 1024, timeout, TEST_CHUNK_SIZE_KB, "/test").await;

        assert!(matches!(result, Err(Error::BodyReadTimeout(_))));
    }

    #[tokio::test]
    async fn extract_body_length_limited_maps_to_payload_too_large() {
        let inner = Body::from(vec![b'x'; 1024]);
        let limited = http_body_util::Limited::new(inner, 128);
        let body = Body::new(limited);
        // Manual limit (4096) is larger than the Limited wrapper's, so only the
        // wrapped body's LengthLimitError can trigger the failure.
        let result = extract_body_with_timeout(body, 4096, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn extract_body_empty() {
        let body = Body::empty();
        let result = extract_body_with_timeout(body, 1024, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(matches!(result, Err(Error::EmptyBody)));
    }

    // --- decompression tests ---

    fn compress_gzip(data: &[u8]) -> Vec<u8> {
        let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
        encoder.write_all(data).unwrap();
        encoder.finish().unwrap()
    }

    fn compress_deflate(data: &[u8]) -> Vec<u8> {
        let mut encoder = DeflateEncoder::new(Vec::new(), Compression::fast());
        encoder.write_all(data).unwrap();
        encoder.finish().unwrap()
    }

    fn compress_zlib(data: &[u8]) -> Vec<u8> {
        let mut encoder = ZlibEncoder::new(Vec::new(), Compression::fast());
        encoder.write_all(data).unwrap();
        encoder.finish().unwrap()
    }

    fn compress_brotli(data: &[u8]) -> Vec<u8> {
        let mut out = Vec::new();
        {
            let mut writer = brotli::CompressorWriter::new(&mut out, 4096, 6, 22);
            writer.write_all(data).unwrap();
        }
        out
    }

    fn compress_zstd(data: &[u8]) -> Vec<u8> {
        zstd::encode_all(std::io::Cursor::new(data), 3).unwrap()
    }

    #[tokio::test]
    async fn decompress_none_is_passthrough() {
        let data = Bytes::from("hello world");
        let result = decompress_payload(None, data.clone(), 4096, TEST_CHUNK_SIZE_KB).await;
        assert_eq!(result.unwrap(), data);
    }

    #[tokio::test]
    async fn decompress_gzip_valid() {
        let original = b"hello gzip world";
        let compressed = Bytes::from(compress_gzip(original));
        let result = decompress_payload(Some("gzip"), compressed, 4096, TEST_CHUNK_SIZE_KB).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn decompress_deflate_raw_valid() {
        let original = b"hello deflate world";
        let compressed = Bytes::from(compress_deflate(original));
        assert_ne!(
            compressed.first(),
            Some(&0x78),
            "raw deflate must not start with zlib magic"
        );
        let result =
            decompress_payload(Some("deflate"), compressed, 4096, TEST_CHUNK_SIZE_KB).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn decompress_deflate_zlib_wrapped_valid() {
        let original = b"hello zlib-wrapped deflate world";
        let compressed = Bytes::from(compress_zlib(original));
        assert_eq!(
            compressed.first(),
            Some(&0x78),
            "zlib payload must start with 0x78 magic"
        );
        let result =
            decompress_payload(Some("deflate"), compressed, 4096, TEST_CHUNK_SIZE_KB).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn decompress_br_valid() {
        let original = b"hello brotli world";
        let compressed = Bytes::from(compress_brotli(original));
        let result = decompress_payload(Some("br"), compressed, 4096, TEST_CHUNK_SIZE_KB).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn decompress_zstd_valid() {
        let original = b"hello zstd world";
        let compressed = Bytes::from(compress_zstd(original));
        let result = decompress_payload(Some("zstd"), compressed, 4096, TEST_CHUNK_SIZE_KB).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn decompress_gzip_exceeds_limit() {
        let big = vec![0u8; 64 * 1024];
        let compressed = Bytes::from(compress_gzip(&big));
        let result = decompress_payload(Some("gzip"), compressed, 1024, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn decompress_deflate_raw_exceeds_limit() {
        let big = vec![0u8; 64 * 1024];
        let compressed = Bytes::from(compress_deflate(&big));
        let result =
            decompress_payload(Some("deflate"), compressed, 1024, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn decompress_deflate_zlib_exceeds_limit() {
        let big = vec![0u8; 64 * 1024];
        let compressed = Bytes::from(compress_zlib(&big));
        assert_eq!(compressed.first(), Some(&0x78));
        let result =
            decompress_payload(Some("deflate"), compressed, 1024, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn decompress_deflate_raw_starting_with_0x78_treated_as_zlib() {
        // Pin the known tradeoff: a raw deflate stream whose first byte is
        // 0x78 gets misrouted to ZlibDecoder and fails. Same strategy as nginx.
        let raw = compress_deflate(b"test payload");
        let mut patched = raw.clone();
        patched[0] = 0x78;
        let compressed = Bytes::from(patched);

        let result =
            decompress_payload(Some("deflate"), compressed, 4096, TEST_CHUNK_SIZE_KB).await;
        assert!(
            result.is_err(),
            "raw deflate starting with 0x78 is misrouted to ZlibDecoder (accepted tradeoff)"
        );
    }

    #[tokio::test]
    async fn decompress_br_exceeds_limit() {
        let big = vec![0u8; 64 * 1024];
        let compressed = Bytes::from(compress_brotli(&big));
        let result = decompress_payload(Some("br"), compressed, 1024, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn decompress_zstd_exceeds_limit() {
        let big = vec![0u8; 64 * 1024];
        let compressed = Bytes::from(compress_zstd(&big));
        let result = decompress_payload(Some("zstd"), compressed, 1024, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn decompress_invalid_gzip_data() {
        let garbage = Bytes::from_static(b"this is not gzip");
        let result = decompress_payload(Some("gzip"), garbage, 4096, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::RequestDecodingError(_))));
    }

    #[tokio::test]
    async fn decompress_invalid_deflate_data() {
        let garbage = Bytes::from_static(b"this is not deflate");
        let result = decompress_payload(Some("deflate"), garbage, 4096, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::RequestDecodingError(_))));
    }

    #[tokio::test]
    async fn decompress_invalid_br_data() {
        let garbage = Bytes::from_static(b"this is not brotli");
        let result = decompress_payload(Some("br"), garbage, 4096, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::RequestDecodingError(_))));
    }

    #[tokio::test]
    async fn decompress_invalid_zstd_data() {
        let garbage = Bytes::from_static(b"this is not zstd");
        let result = decompress_payload(Some("zstd"), garbage, 4096, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::RequestDecodingError(_))));
    }

    #[tokio::test]
    async fn decompress_unsupported_encoding() {
        let data = Bytes::from("irrelevant");
        let result = decompress_payload(Some("lz4"), data, 4096, TEST_CHUNK_SIZE_KB).await;
        assert!(matches!(result, Err(Error::UnsupportedEncoding(_))));
    }

    // --- split-limit tests (compressed limit < decompressed limit) ---

    /// Simulates the two-step handler flow: extract with a compressed limit,
    /// then decompress with a (larger) decompressed limit.
    async fn handler_roundtrip(
        encoding: Option<&str>,
        raw: Bytes,
        compressed_limit: usize,
        decompressed_limit: usize,
    ) -> Result<Bytes, Error> {
        let body = Body::from(raw);
        let wire =
            extract_body_with_timeout(body, compressed_limit, None, TEST_CHUNK_SIZE_KB, "/test")
                .await?;
        decompress_payload(encoding, wire, decompressed_limit, TEST_CHUNK_SIZE_KB).await
    }

    #[tokio::test]
    async fn extract_body_at_exact_limit_succeeds() {
        let data = vec![b'x'; 128];
        let body = Body::from(data.clone());
        let result = extract_body_with_timeout(body, 128, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert_eq!(result.unwrap().len(), 128);
    }

    #[tokio::test]
    async fn extract_body_one_byte_over_limit_rejected() {
        let data = vec![b'x'; 129];
        let body = Body::from(data);
        let result = extract_body_with_timeout(body, 128, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(matches!(result, Err(Error::PayloadTooLarge(_))));
    }

    #[tokio::test]
    async fn split_limits_gzip_bomb_rejected_at_decompression() {
        let big = vec![0u8; 8192];
        let compressed = Bytes::from(compress_gzip(&big));
        assert!(
            compressed.len() < 256,
            "compressed should be much smaller than decompressed for this test"
        );
        let result = handler_roundtrip(Some("gzip"), compressed, 256, 4096).await;
        assert!(
            matches!(result, Err(Error::PayloadTooLarge(_))),
            "decompressed 8KB should exceed the 4KB decompressed limit"
        );
    }

    #[tokio::test]
    async fn split_limits_zstd_bomb_rejected_at_decompression() {
        let big = vec![0u8; 8192];
        let compressed = Bytes::from(compress_zstd(&big));
        assert!(
            compressed.len() < 256,
            "compressed should be much smaller than decompressed for this test"
        );
        let result = handler_roundtrip(Some("zstd"), compressed, 256, 4096).await;
        assert!(
            matches!(result, Err(Error::PayloadTooLarge(_))),
            "decompressed 8KB should exceed the 4KB decompressed limit"
        );
    }

    #[tokio::test]
    async fn split_limits_compressed_over_wire_limit_rejected() {
        let big = vec![b'a'; 4096];
        let compressed = Bytes::from(compress_gzip(&big));
        let wire_limit = compressed.len() - 1;
        let result = handler_roundtrip(Some("gzip"), compressed, wire_limit, 1024 * 1024).await;
        assert!(
            matches!(result, Err(Error::PayloadTooLarge(_))),
            "wire bytes should exceed the compressed limit"
        );
    }

    #[tokio::test]
    async fn split_limits_both_within_bounds_succeeds_gzip() {
        let original = b"split limit roundtrip gzip";
        let compressed = Bytes::from(compress_gzip(original));
        let result = handler_roundtrip(Some("gzip"), compressed, 4096, 4096).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn split_limits_both_within_bounds_succeeds_zstd() {
        let original = b"split limit roundtrip zstd";
        let compressed = Bytes::from(compress_zstd(original));
        let result = handler_roundtrip(Some("zstd"), compressed, 4096, 4096).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn split_limits_both_within_bounds_succeeds_deflate() {
        let original = b"split limit roundtrip deflate";
        let compressed = Bytes::from(compress_deflate(original));
        let result = handler_roundtrip(Some("deflate"), compressed, 4096, 4096).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn split_limits_both_within_bounds_succeeds_brotli() {
        let original = b"split limit roundtrip brotli";
        let compressed = Bytes::from(compress_brotli(original));
        let result = handler_roundtrip(Some("br"), compressed, 4096, 4096).await;
        assert_eq!(result.unwrap(), Bytes::from_static(original));
    }

    #[tokio::test]
    async fn split_limits_uncompressed_passthrough_within_bounds() {
        let original = Bytes::from("no encoding, just plain text");
        let result = handler_roundtrip(None, original.clone(), 4096, 4096).await;
        assert_eq!(result.unwrap(), original);
    }

    #[tokio::test]
    async fn extract_body_stream_error_returns_decoding_error() {
        let error_stream = stream::once(async {
            Err::<Bytes, std::io::Error>(std::io::Error::new(
                std::io::ErrorKind::ConnectionReset,
                "simulated stream error",
            ))
        });
        let body = Body::from_stream(error_stream);
        let result = extract_body_with_timeout(body, 4096, None, TEST_CHUNK_SIZE_KB, "/test").await;
        assert!(
            matches!(result, Err(Error::RequestDecodingError(_))),
            "stream error should surface as RequestDecodingError, got: {result:?}"
        );
    }
}