Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): bound decompressed request bodies to close a zip-bomb DoS (#3325)

Tejas Chopra25a4e71b295f
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 8d2a2f43c..fc57686a3 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -1062,6 +1062,18 @@ def append_text_to_latest_user_input_item(
# Maximum request body size (100MB - increased to support image-heavy requests)
MAX_REQUEST_BODY_SIZE = 100 * 1024 * 1024
+# A *decompressed* body obeys the same ceiling as an uncompressed one. The
+# Content-Length gate at the handlers only ever saw the compressed wire size, so
+# a client could buy unlimited extra capacity with a compression ratio: ~2MB of
+# zeros expands to 2GB and the process dies allocating it (#3284). Same number,
+# deliberately: nobody should get more room by arriving gzipped.
+MAX_DECOMPRESSED_BODY_SIZE = MAX_REQUEST_BODY_SIZE
+
+# How much decompressed output to pull per step. The cap is re-checked after
+# every chunk, so the peak allocation is the limit plus one chunk — never the
+# full expansion of a bomb.
+_DECOMPRESS_CHUNK_SIZE = 64 * 1024
+
# Maximum SSE buffer size (10MB - prevents memory exhaustion from malformed streams)
MAX_SSE_BUFFER_SIZE = 10 * 1024 * 1024
@@ -2537,24 +2549,145 @@ def apply_session_sticky_ccr_tool(
return tools_out, True
+class RequestBodyTooLarge(ValueError):
+ """A decompressed request body exceeded :data:`MAX_DECOMPRESSED_BODY_SIZE`.
+
+ Subclasses ``ValueError`` so every existing ``except ValueError`` call site
+ keeps answering 400 unchanged, while giving a caller that would rather
+ answer 413 a type to branch on.
+ """
+
+
+def _inflate_bounded(raw: bytes, *, wbits: int, label: str, multi_member: bool = False) -> bytes:
+ """Incrementally inflate ``raw``, stopping the instant output passes the cap.
+
+ ``zlib.decompress``/``gzip.decompress`` materialize the whole expansion
+ before anything can inspect it, which is what makes a bomb fatal. Feeding
+ the stream through ``decompressobj`` with a ``max_length`` keeps unproduced
+ output in ``unconsumed_tail`` instead of memory, so the check below runs
+ before the allocation rather than after it.
+ """
+ import zlib
+
+ out = bytearray()
+ pending = raw
+ first_member = True
+ while True:
+ if multi_member and not first_member:
+ # gzip is a *sequence* of members and CPython's reader skips NUL
+ # padding between and after them — real clients emit it, and
+ # `gzip.decompress(member + b"\x00" * 16)` returns the payload
+ # rather than raising. Parsing that padding as a fresh member
+ # would reject bodies the one-shot call accepted.
+ pending = pending.lstrip(b"\x00")
+ if multi_member and not pending:
+ # Either an empty body (`gzip.decompress(b"")` == b"") or a clean
+ # end after the last member. A body that is *only* padding never
+ # reaches here: `first_member` is still True, so it falls through
+ # to the header parse below and fails there, as CPython does.
+ break
+ decompressor = zlib.decompressobj(wbits)
+ while True:
+ chunk = decompressor.decompress(pending, _DECOMPRESS_CHUNK_SIZE)
+ out += chunk
+ if len(out) > MAX_DECOMPRESSED_BODY_SIZE:
+ raise RequestBodyTooLarge(
+ f"Decompressed {label} request body exceeds "
+ f"{MAX_DECOMPRESSED_BODY_SIZE // (1024 * 1024)}MB"
+ )
+ pending = decompressor.unconsumed_tail
+ if decompressor.eof or not pending:
+ break
+ if not chunk:
+ # Input left over but nothing produced: the stream cannot
+ # advance, and looping again would spin forever.
+ raise ValueError(f"Failed to decompress {label} request body: stalled stream")
+ out += decompressor.flush()
+ if len(out) > MAX_DECOMPRESSED_BODY_SIZE:
+ raise RequestBodyTooLarge(
+ f"Decompressed {label} request body exceeds "
+ f"{MAX_DECOMPRESSED_BODY_SIZE // (1024 * 1024)}MB"
+ )
+ if not decompressor.eof:
+ raise ValueError(f"Failed to decompress {label} request body: truncated stream")
+ # gzip streams may carry several members; `gzip.decompress` concatenates
+ # them, so restart on the trailer to keep that behavior.
+ pending = decompressor.unused_data
+ first_member = False
+ if not multi_member:
+ break
+ return bytes(out)
+
+
+def _zstd_bounded(raw: bytes) -> bytes:
+ """Read a zstd frame in chunks so a high-ratio frame cannot outrun the cap."""
+ import zstandard
+
+ out = bytearray()
+ dctx = zstandard.ZstdDecompressor()
+ with dctx.stream_reader(raw) as reader:
+ while True:
+ chunk = reader.read(_DECOMPRESS_CHUNK_SIZE)
+ if not chunk:
+ break
+ out += chunk
+ if len(out) > MAX_DECOMPRESSED_BODY_SIZE:
+ raise RequestBodyTooLarge(
+ f"Decompressed zstd request body exceeds "
+ f"{MAX_DECOMPRESSED_BODY_SIZE // (1024 * 1024)}MB"
+ )
+ return bytes(out)
+
+
+def _brotli_bounded(raw: bytes) -> bytes:
+ """Feed brotli input in slices, checking the cap after each one."""
+ import brotli
+
+ decompressor_cls = getattr(brotli, "Decompressor", None)
+ if decompressor_cls is None:
+ # Fail closed rather than fall back to the unbounded one-shot call: an
+ # old library is not a reason to reopen the hole.
+ raise ValueError(
+ "Installed 'brotli' is too old for incremental decompression "
+ "(needs brotli.Decompressor); upgrade it to accept br request bodies."
+ )
+
+ decompressor = decompressor_cls()
+ out = bytearray()
+ for start in range(0, len(raw), _DECOMPRESS_CHUNK_SIZE):
+ out += decompressor.process(raw[start : start + _DECOMPRESS_CHUNK_SIZE])
+ if len(out) > MAX_DECOMPRESSED_BODY_SIZE:
+ raise RequestBodyTooLarge(
+ f"Decompressed brotli request body exceeds "
+ f"{MAX_DECOMPRESSED_BODY_SIZE // (1024 * 1024)}MB"
+ )
+ is_finished = getattr(decompressor, "is_finished", None)
+ if is_finished is not None and not is_finished():
+ raise ValueError("Failed to decompress brotli request body: truncated stream")
+ return bytes(out)
+
+
async def _read_request_body_bytes(request: Request) -> bytes:
"""Read and (if needed) decompress the request body, returning raw UTF-8 bytes.
Mirrors ``_read_request_json`` but returns the bytes pre-parse so
forwarders can implement byte-faithful passthrough (PR-A3, fixes P0-2).
- Raises ``ValueError`` on any decompression failure.
+ Raises ``ValueError`` on any decompression failure, and the
+ :class:`RequestBodyTooLarge` subclass when the *decompressed* body would
+ exceed :data:`MAX_DECOMPRESSED_BODY_SIZE`.
"""
encoding = (request.headers.get("content-encoding") or "").lower().strip()
raw = await request.body()
+ # Every branch below decompresses incrementally against
+ # MAX_DECOMPRESSED_BODY_SIZE. RequestBodyTooLarge is re-raised ahead of the
+ # generic handlers so the size refusal is not reworded into a vague
+ # "failed to decompress" (#3284).
if encoding in ("zstd", "zstandard"):
try:
- import zstandard
-
- dctx = zstandard.ZstdDecompressor()
- reader = dctx.stream_reader(raw)
- raw = reader.read()
- reader.close()
+ raw = _zstd_bounded(raw)
+ except RequestBodyTooLarge:
+ raise
except ImportError:
raise ValueError(
"Request body is zstd-compressed but the 'zstandard' package is not installed. "
@@ -2563,24 +2696,30 @@ async def _read_request_body_bytes(request: Request) -> bytes:
except Exception as exc:
raise ValueError(f"Failed to decompress zstd request body: {exc}") from exc
elif encoding == "gzip":
- import gzip as _gzip
+ import zlib
try:
- raw = _gzip.decompress(raw)
+ raw = _inflate_bounded(raw, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True)
+ except ValueError:
+ # Covers RequestBodyTooLarge and the explicit stream diagnostics,
+ # both already carrying the message we want.
+ raise
except Exception as exc:
raise ValueError(f"Failed to decompress gzip request body: {exc}") from exc
elif encoding == "deflate":
import zlib
try:
- raw = zlib.decompress(raw)
+ raw = _inflate_bounded(raw, wbits=zlib.MAX_WBITS, label="deflate")
+ except ValueError:
+ raise
except Exception as exc:
raise ValueError(f"Failed to decompress deflate request body: {exc}") from exc
elif encoding == "br":
try:
- import brotli
-
- raw = brotli.decompress(raw)
+ raw = _brotli_bounded(raw)
+ except ValueError:
+ raise
except ImportError:
raise ValueError(
"Request body is brotli-compressed but the 'brotli' package is not installed."
diff --git a/tests/test_request_body_decompression_limits.py b/tests/test_request_body_decompression_limits.py
new file mode 100644
index 000000000..3b4e1be7d
--- /dev/null
+++ b/tests/test_request_body_decompression_limits.py
@@ -0,0 +1,300 @@
+"""Decompressed request bodies must obey the same size ceiling as plain ones.
+
+The handlers gate on ``Content-Length`` — the *compressed* wire size — so a
+high-ratio body bought unlimited capacity: ~2MB of zeros expands to 2GB and the
+process dies allocating it, from a single unauthenticated request (#3284).
+
+The property under test is not merely "an oversized body is rejected" but
+"it is rejected *without being materialized first*" — a guard that checks the
+size after a one-shot ``decompress()`` has already OOM'd. So the memory tests
+below assert on peak allocation, not just on the exception.
+
+Bombs are built by streaming a compressor, so the fixtures themselves stay
+small; the cap is monkeypatched down so the tests neither allocate nor scan
+100MB to prove the point.
+"""
+
+from __future__ import annotations
+
+import gzip
+import io
+import tracemalloc
+import zlib
+
+import pytest
+
+
+def _helpers():
+ """Resolve the helpers module at call time, not at import time.
+
+ Other suites swap ``headroom.proxy`` in and out of ``sys.modules`` for their
+ own isolation, which can leave two live copies of this module — and two
+ distinct ``RequestBodyTooLarge`` classes, so a class captured at import time
+ stops matching the one actually raised. Production never reimports this way,
+ and callers catch the builtin ``ValueError`` regardless, so this is a test
+ concern only. Resolving through ``sys.modules`` keeps the helper and the
+ exception it raises on the same module object.
+ """
+ import importlib
+
+ return importlib.import_module("headroom.proxy.helpers")
+
+
+MB = 1024 * 1024
+PAYLOAD = b'{"model":"claude-sonnet-5","messages":[{"role":"user","content":"hi"}]}'
+
+# Expands to 64MB. Every test that uses it caps well below that, so the refusal
+# has to happen mid-stream rather than at the end.
+BOMB_PLAIN_SIZE = 64 * MB
+SMALL_CAP = 1 * MB
+
+
+def _gzip_bomb(total: int = BOMB_PLAIN_SIZE) -> bytes:
+ """A gzip stream expanding to ``total`` bytes, built without holding them."""
+ compressor = zlib.compressobj(9, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+ buf = io.BytesIO()
+ chunk = b"\x00" * MB
+ for _ in range(total // MB):
+ buf.write(compressor.compress(chunk))
+ buf.write(compressor.flush())
+ return buf.getvalue()
+
+
+def _deflate_bomb(total: int = BOMB_PLAIN_SIZE) -> bytes:
+ compressor = zlib.compressobj(9)
+ buf = io.BytesIO()
+ chunk = b"\x00" * MB
+ for _ in range(total // MB):
+ buf.write(compressor.compress(chunk))
+ buf.write(compressor.flush())
+ return buf.getvalue()
+
+
+class _Request:
+ """Minimal stand-in for the Starlette Request the reader actually takes."""
+
+ def __init__(self, body: bytes, content_encoding: str = "") -> None:
+ self._body = body
+ self.headers = {"content-encoding": content_encoding}
+
+ async def body(self) -> bytes:
+ return self._body
+
+
+def small_cap(monkeypatch: pytest.MonkeyPatch) -> int:
+ monkeypatch.setattr(_helpers(), "MAX_DECOMPRESSED_BODY_SIZE", SMALL_CAP)
+ return SMALL_CAP
+
+
+# ───────────────────────────── the ceiling itself ──────────────────────────
+
+
+def test_decompressed_ceiling_matches_the_plain_body_ceiling() -> None:
+ """A client must not get more room by arriving compressed."""
+ assert _helpers().MAX_DECOMPRESSED_BODY_SIZE == _helpers().MAX_REQUEST_BODY_SIZE
+
+
+def test_too_large_is_a_value_error() -> None:
+ """Every existing call site catches ValueError; the subclass must not escape it."""
+ assert issubclass(_helpers().RequestBodyTooLarge, ValueError)
+
+
+# ───────────────────────────── refusal, per codec ──────────────────────────
+
+
+def test_gzip_bomb_is_refused(small_cap: int) -> None:
+ with pytest.raises(_helpers().RequestBodyTooLarge):
+ _helpers()._inflate_bounded(
+ _gzip_bomb(), wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+
+
+def test_deflate_bomb_is_refused(small_cap: int) -> None:
+ with pytest.raises(_helpers().RequestBodyTooLarge):
+ _helpers()._inflate_bounded(_deflate_bomb(), wbits=zlib.MAX_WBITS, label="deflate")
+
+
+def test_zstd_bomb_is_refused(small_cap: int) -> None:
+ zstandard = pytest.importorskip("zstandard")
+ bomb = zstandard.ZstdCompressor(level=19).compress(b"\x00" * BOMB_PLAIN_SIZE)
+ with pytest.raises(_helpers().RequestBodyTooLarge):
+ _helpers()._zstd_bounded(bomb)
+
+
+def test_brotli_bomb_is_refused(small_cap: int) -> None:
+ brotli = pytest.importorskip("brotli")
+ bomb = brotli.compress(b"\x00" * BOMB_PLAIN_SIZE)
+ with pytest.raises(_helpers().RequestBodyTooLarge):
+ _helpers()._brotli_bounded(bomb)
+
+
+# ─────────────────────── refused *before* materializing ────────────────────
+
+
+ ("make_bomb", "call"),
+ [
+ (
+ _gzip_bomb,
+ lambda b: _helpers()._inflate_bounded(
+ b, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ ),
+ ),
+ (
+ _deflate_bomb,
+ lambda b: _helpers()._inflate_bounded(b, wbits=zlib.MAX_WBITS, label="deflate"),
+ ),
+ ],
+)
+def test_bomb_never_materializes(make_bomb, call, small_cap: int) -> None:
+ """Peak allocation stays near the cap, not near the expansion.
+
+ This is the whole fix: a check placed after a one-shot decompress() would
+ pass the refusal assertions above while still OOM-ing the process.
+ """
+ bomb = make_bomb()
+
+ already_tracing = tracemalloc.is_tracing()
+ if not already_tracing:
+ tracemalloc.start()
+ try:
+ # reset_peak so a peak set by whatever ran before this test cannot be
+ # mistaken for the bomb materializing here.
+ tracemalloc.reset_peak()
+ with pytest.raises(_helpers().RequestBodyTooLarge):
+ call(bomb)
+ _, peak = tracemalloc.get_traced_memory()
+ finally:
+ if not already_tracing:
+ tracemalloc.stop()
+
+ # Generous headroom over the 1MB cap; the unbounded path would peak near
+ # the 64MB expansion (and higher still, with the copy decompress() makes).
+ assert peak < 8 * MB, f"peak {peak / MB:.1f}MB suggests the bomb was materialized"
+
+
+# ─────────────────────────── ordinary bodies still work ────────────────────
+
+
+def test_gzip_round_trips() -> None:
+ assert (
+ _helpers()._inflate_bounded(
+ gzip.compress(PAYLOAD), wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+ == PAYLOAD
+ )
+
+
+def test_deflate_round_trips() -> None:
+ assert (
+ _helpers()._inflate_bounded(zlib.compress(PAYLOAD), wbits=zlib.MAX_WBITS, label="deflate")
+ == PAYLOAD
+ )
+
+
+def test_zstd_round_trips() -> None:
+ zstandard = pytest.importorskip("zstandard")
+ assert _helpers()._zstd_bounded(zstandard.ZstdCompressor().compress(PAYLOAD)) == PAYLOAD
+
+
+def test_brotli_round_trips() -> None:
+ brotli = pytest.importorskip("brotli")
+ assert _helpers()._brotli_bounded(brotli.compress(PAYLOAD)) == PAYLOAD
+
+
+def test_multi_member_gzip_is_still_concatenated() -> None:
+ """`gzip.decompress` joins members; a single-member decompressobj would not."""
+ stream = gzip.compress(b'{"a":1}') + gzip.compress(b'{"b":2}')
+
+ assert (
+ _helpers()._inflate_bounded(
+ stream, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+ == b'{"a":1}{"b":2}'
+ )
+
+
+def test_gzip_trailing_nul_padding_is_tolerated() -> None:
+ """Real clients pad gzip bodies with NULs and `gzip.decompress` accepts it.
+
+ A member-restart loop that treats the padding as a fresh member rejects
+ bodies the one-shot call happily decoded — a silent compatibility break
+ that no bomb test would have caught.
+ """
+ padded = gzip.compress(PAYLOAD) + b"\x00" * 16
+
+ assert gzip.decompress(padded) == PAYLOAD # the behavior being matched
+ assert (
+ _helpers()._inflate_bounded(
+ padded, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+ == PAYLOAD
+ )
+
+
+def test_gzip_padding_between_members_is_skipped() -> None:
+ stream = gzip.compress(b'{"a":1}') + b"\x00" * 8 + gzip.compress(b'{"b":2}')
+
+ assert (
+ _helpers()._inflate_bounded(
+ stream, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+ == b'{"a":1}{"b":2}'
+ )
+
+
+def test_empty_gzip_body_matches_the_one_shot_call() -> None:
+ """`gzip.decompress(b"")` returns b""; refusing it would be a new rejection."""
+ assert gzip.decompress(b"") == b""
+ assert (
+ _helpers()._inflate_bounded(b"", wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True)
+ == b""
+ )
+
+
+def test_gzip_body_of_only_padding_is_still_rejected() -> None:
+ """Tolerating padding must not turn a bodyless run of NULs into success."""
+ with pytest.raises(gzip.BadGzipFile):
+ gzip.decompress(b"\x00" * 8)
+ # zlib.error, not ValueError: only `_read_request_body_bytes` wraps the
+ # codec's own error, and it turns this into the same 400 as before.
+ with pytest.raises(zlib.error):
+ _helpers()._inflate_bounded(
+ b"\x00" * 8, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+
+
+def test_truncated_gzip_still_errors() -> None:
+ """Losing the one-shot call must not turn a corrupt body into a silent empty one."""
+ truncated = gzip.compress(PAYLOAD)[:-5]
+
+ with pytest.raises(ValueError):
+ _helpers()._inflate_bounded(
+ truncated, wbits=16 + zlib.MAX_WBITS, label="gzip", multi_member=True
+ )
+
+
+# ──────────────────────────── through the entry point ──────────────────────
+
+
+async def test_reader_refuses_a_bomb(small_cap: int) -> None:
+ with pytest.raises(_helpers().RequestBodyTooLarge):
+ await _helpers()._read_request_body_bytes(_Request(_gzip_bomb(), "gzip"))
+
+
+async def test_reader_passes_an_ordinary_compressed_body(small_cap: int) -> None:
+ assert (
+ await _helpers()._read_request_body_bytes(_Request(gzip.compress(PAYLOAD), "gzip"))
+ == PAYLOAD
+ )
+
+
+async def test_reader_leaves_uncompressed_bodies_alone(small_cap: int) -> None:
+ assert await _helpers()._read_request_body_bytes(_Request(PAYLOAD, "")) == PAYLOAD
+ assert await _helpers()._read_request_body_bytes(_Request(PAYLOAD, "identity")) == PAYLOAD
+
+
+async def test_reader_still_rejects_an_unknown_encoding() -> None:
+ with pytest.raises(ValueError, match="Unsupported Content-Encoding"):
+ await _helpers()._read_request_body_bytes(_Request(PAYLOAD, "snappy"))