Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): run extension middleware inside the security gate and body ceiling (#3847)

Tejas Chopra1854fd7fd3f5
diff --git a/headroom/proxy/extensions.py b/headroom/proxy/extensions.py
index 183028f71..d7606a37e 100644
--- a/headroom/proxy/extensions.py
+++ b/headroom/proxy/extensions.py
@@ -18,6 +18,30 @@ Each ``install`` callable is invoked with the FastAPI ``app`` and the
OSS makes no assumptions about what extensions do. The interface is
deliberately minimal; extensions own the complexity behind it.
+Where extension middleware sits
+-------------------------------
+
+``install_all`` runs **before** the proxy registers its inbound security gate
+and its request-body ceiling, and Starlette builds the stack so that the last
+layer registered is the outermost. Middleware an extension adds therefore
+always runs *inside* both, and an extension author can rely on two things
+without re-implementing them:
+
+ * a request that reaches extension middleware has already passed
+ ``HEADROOM_PROXY_TOKEN`` (when one is configured) — an extension never sees,
+ answers, buffers, or rewrites headers on an unauthenticated request, on
+ either transport (HTTP or WebSocket);
+ * a request body that reaches extension middleware is already bounded by
+ :data:`headroom.proxy.helpers.MAX_REQUEST_BODY_SIZE` — an extension that
+ buffers the body cannot be made to hold more than that.
+
+The flip side is a constraint: extension middleware cannot act as the
+proxy's authentication layer, because the proxy's own gate runs first. An
+extension that needs its own inbound auth applies it in addition to, not
+instead of, the proxy token. ``install_all`` records every middleware entry an
+extension adds on ``app.state.extension_middleware`` so the ordering can be
+asserted by tests rather than assumed.
+
Reporting what an extension saved, and what it cost
---------------------------------------------------
@@ -164,9 +188,17 @@ def install_all(
wildcard = "*" in enabled_set
installed: list[str] = []
failed: list[str] = []
+ # Middleware entries added by extensions, in registration order. Starlette
+ # prepends on ``add_middleware``, so the entries an install() adds are
+ # whatever is new at the FRONT of ``app.user_middleware`` afterwards. Kept
+ # on ``app.state`` so the ordering contract in the module docstring is
+ # something the test suite can check against the real app.
+ middleware_list = getattr(app, "user_middleware", None)
+ added_middleware: list[Any] = []
for name, install in discovered:
if not wildcard and name not in enabled_set:
continue
+ before = len(middleware_list) if isinstance(middleware_list, list) else 0
try:
install(app, config)
except Exception as exc: # noqa: BLE001 — one bad extension must not brick the proxy
@@ -182,8 +214,17 @@ def install_all(
failed.append(name)
continue
installed.append(name)
+ if isinstance(middleware_list, list):
+ added_middleware.extend(middleware_list[: len(middleware_list) - before])
log.info("proxy extension installed: %s", name)
+ state = getattr(app, "state", None)
+ if state is not None:
+ try:
+ state.extension_middleware = added_middleware
+ except Exception: # noqa: BLE001 — a bare test double may not accept attributes
+ log.debug("could not record extension middleware on app.state", exc_info=True)
+
if failed:
skipped = ",".join(sorted(failed))
log.warning("proxy extensions skipped due to install errors: %s", skipped)
diff --git a/headroom/proxy/request_body_limit.py b/headroom/proxy/request_body_limit.py
new file mode 100644
index 000000000..9a6a7a5c9
--- /dev/null
+++ b/headroom/proxy/request_body_limit.py
@@ -0,0 +1,182 @@
+"""Request-body ceiling enforced as a raw ASGI layer.
+
+The proxy already bounds request bodies inside its handlers: every route that
+reads JSON goes through ``helpers._read_request_body_bytes``, which refuses
+anything over :data:`headroom.proxy.helpers.MAX_REQUEST_BODY_SIZE` while it is
+still streaming in. That protects the handlers, and only the handlers. Anything
+that reads the body *before* a handler runs, which in practice means an ASGI
+middleware installed by a proxy extension, never sees that check. An extension
+that buffers the body to inspect or rewrite it (a router that reads
+``service_tier``, a redactor, a relay) therefore holds an unbounded body for
+as long as the client cares to send one.
+
+This module makes the ceiling a property of the ASGI stack instead of a
+convention every extension has to know about. :class:`RequestBodyLimitMiddleware`
+is registered by :func:`headroom.proxy.server.create_app` immediately inside
+the inbound security gate and *outside* every extension, so the guarantee an
+extension author gets is:
+
+* an unauthenticated request never reaches extension code (the gate), and
+* an authenticated request whose body exceeds the ceiling is refused with the
+ same status and payload the handlers use, before any extension buffers it.
+
+Enforcement is the same two-step the handlers use: a declared
+``Content-Length`` above the ceiling is refused before the body is read at all,
+and the body stream itself is metered so an absent or understated
+``Content-Length`` (chunked transfer) is bounded too. Once the stream trips the
+ceiling the client gets the refusal, the application's own attempt to respond
+is discarded, and :class:`~headroom.proxy.helpers.RequestBodyTooLarge` is
+raised into whatever was reading the stream so it stops.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from collections.abc import Awaitable, Callable, MutableMapping
+from typing import Any
+
+from headroom.proxy import helpers as _helpers
+
+logger = logging.getLogger("headroom.proxy")
+
+Scope = MutableMapping[str, Any]
+Receive = Callable[[], Awaitable[MutableMapping[str, Any]]]
+Send = Callable[[MutableMapping[str, Any]], Awaitable[None]]
+
+# Paths whose clients speak the Anthropic error dialect. Everything else gets
+# the OpenAI-style payload, which is what the handlers' own 413s use.
+_ANTHROPIC_PATH_MARKERS = ("/v1/messages",)
+
+
+def _too_large_payload(path: str, limit: int) -> bytes:
+ """Mirror the handlers' own 413 payloads so clients see one shape per dialect."""
+ message = f"Request body too large. Maximum size is {limit // (1024 * 1024)}MB"
+ if any(marker in path for marker in _ANTHROPIC_PATH_MARKERS):
+ payload: dict[str, Any] = {
+ "type": "error",
+ "error": {"type": "request_too_large", "message": message},
+ }
+ else:
+ payload = {
+ "error": {
+ "message": message,
+ "type": "invalid_request_error",
+ "code": "request_too_large",
+ }
+ }
+ return json.dumps(payload).encode("utf-8")
+
+
+def _declared_content_length(scope: Scope) -> int | None:
+ for name, value in scope.get("headers") or ():
+ if name.lower() == b"content-length":
+ try:
+ return int(value.decode("latin-1").strip())
+ except ValueError:
+ return None
+ return None
+
+
+class RequestBodyLimitMiddleware:
+ """Refuse HTTP request bodies above the proxy's ceiling before anything buffers them.
+
+ ``max_bytes`` defaults to :data:`headroom.proxy.helpers.MAX_REQUEST_BODY_SIZE`,
+ resolved per request so a test (or a future policy knob) that adjusts the
+ module constant is honoured without rebuilding the app. Non-HTTP scopes
+ (WebSocket, lifespan) pass straight through: WebSocket frames have their own
+ limits in the relay handlers.
+ """
+
+ def __init__(self, app: Any, *, max_bytes: int | None = None) -> None:
+ self.app = app
+ self._max_bytes = max_bytes
+
+ @property
+ def max_bytes(self) -> int:
+ if self._max_bytes is not None:
+ return self._max_bytes
+ return int(_helpers.MAX_REQUEST_BODY_SIZE)
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ if scope.get("type") != "http":
+ await self.app(scope, receive, send)
+ return
+
+ limit = self.max_bytes
+ path = str(scope.get("path") or "")
+
+ declared = _declared_content_length(scope)
+ if declared is not None and declared > limit:
+ logger.warning(
+ "event=request_body_rejected path=%s reason=content_length declared=%d limit=%d",
+ path,
+ declared,
+ limit,
+ )
+ await self._reject(send, path, limit)
+ return
+
+ # ``received`` meters the stream; ``responded`` records that *we* have
+ # answered, after which anything the application tries to send is
+ # discarded (it is reacting to a body we cut off); ``started`` records
+ # that the application answered first, in which case it is too late
+ # for us to substitute a refusal and we only stop feeding the body.
+ state = {"received": 0, "responded": False, "started": False}
+
+ async def metered_receive() -> MutableMapping[str, Any]:
+ message = await receive()
+ if message.get("type") == "http.request":
+ state["received"] += len(message.get("body") or b"")
+ if state["received"] > limit:
+ logger.warning(
+ "event=request_body_rejected path=%s reason=stream received>%d limit=%d",
+ path,
+ limit,
+ limit,
+ )
+ if not state["started"] and not state["responded"]:
+ state["responded"] = True
+ await self._reject(send, path, limit)
+ raise _helpers.RequestBodyTooLarge(
+ f"Request body exceeds {limit // (1024 * 1024)}MB"
+ )
+ return message
+
+ async def guarded_send(message: MutableMapping[str, Any]) -> None:
+ if state["responded"]:
+ return
+ if message.get("type") == "http.response.start":
+ state["started"] = True
+ await send(message)
+
+ try:
+ await self.app(scope, metered_receive, guarded_send)
+ except _helpers.RequestBodyTooLarge:
+ # Raised by metered_receive above, or by a handler's own bounded
+ # read that nothing caught. Either way the answer is the same 413;
+ # if the application already started a response there is nothing
+ # left to say on the wire.
+ if state["responded"] or state["started"]:
+ return
+ state["responded"] = True
+ await self._reject(send, path, limit)
+
+ @staticmethod
+ async def _reject(send: Send, path: str, limit: int) -> None:
+ body = _too_large_payload(path, limit)
+ await send(
+ {
+ "type": "http.response.start",
+ "status": _helpers.get_body_too_large_status(),
+ "headers": [
+ (b"content-type", b"application/json"),
+ (b"content-length", str(len(body)).encode("ascii")),
+ (b"cache-control", b"no-store"),
+ ],
+ }
+ )
+ await send({"type": "http.response.body", "body": body})
+
+
+__all__ = ["RequestBodyLimitMiddleware"]
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 11ca2f9fc..44e21ce23 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -179,6 +179,7 @@ from headroom.proxy.project_context import (
)
from headroom.proxy.prometheus_metrics import PrometheusMetrics # noqa: F401
from headroom.proxy.rate_limiter import TokenBucketRateLimiter # noqa: F401
+from headroom.proxy.request_body_limit import RequestBodyLimitMiddleware
from headroom.proxy.request_logger import RequestLogger # noqa: F401
from headroom.proxy.savings_tracker import LITELLM_AVAILABLE
from headroom.proxy.semantic_cache import SemanticCache # noqa: F401
@@ -3816,6 +3817,21 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
# headers, and an audit trail for state-mutating admin endpoints.
# WebSocket handshakes are covered separately — see the
# WebSocketAuthMiddleware registration just below this block.
+ #
+ # Ordering contract. Starlette's ``add_middleware`` prepends, so the LAST
+ # layer registered is the OUTERMOST at request time. Everything below is
+ # therefore registered inner-to-outer, and the order is load-bearing:
+ #
+ # 1. third-party extensions (``install_all``) — innermost of the three
+ # 2. RequestBodyLimitMiddleware — bounds bodies before
+ # any extension buffers one
+ # 3. _security_gate + WebSocketAuthMiddleware — outermost: auth first
+ #
+ # Extensions used to install *after* the gate, which made their middleware
+ # the outermost layer: an extension that answered a request itself, or
+ # buffered its body, or rewrote ``Authorization``, did so before the proxy
+ # token was ever checked. ``tests/test_proxy_extension_ordering.py`` pins
+ # the order; ``extensions.py`` documents it for extension authors.
_proxy_token = config.proxy_token or os.environ.get("HEADROOM_PROXY_TOKEN") or None
# Pre-encode once for constant-time comparison (compare_digest on str raises
# TypeError for non-ASCII input, which would turn a 401 into a 500).
@@ -3845,6 +3861,32 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
"Strict-Transport-Security", "max-age=31536000; includeSubDomains"
)
+ # Third-party proxy extensions (Enterprise, custom plugins). Discovered via
+ # the `headroom.proxy_extension` entry-point group, but **opt-in only**:
+ # only names listed in config.proxy_extensions (CLI: --proxy-extension,
+ # env: HEADROOM_PROXY_EXTENSIONS) actually get installed. Discovery alone
+ # never runs third-party code. An extension that raises from its install()
+ # disables *itself*: `install_all` logs it, prints a SKIPPED line to stderr
+ # and the proxy keeps serving without that extension (extensions.py).
+ # One bad plugin must not brick the proxy. Note the consequence: a plugin
+ # whose licence gate raises is absent, not fatal — the feature fails closed,
+ # the process does not.
+ #
+ # Installed HERE, before the body limit and the security gate are
+ # registered, so any middleware an extension adds sits INSIDE both (see the
+ # ordering contract above). Routes an extension registers are unaffected
+ # by position. ``_proxy_token`` was read above, before extensions ran, so
+ # an extension cannot change which credential the gate enforces.
+ from headroom.proxy.extensions import install_all as _install_extensions
+
+ _install_extensions(app, config, enabled=getattr(config, "proxy_extensions", None))
+
+ # Body ceiling for everything inside it (extensions and handlers alike).
+ # The handlers keep their own streaming check — this layer exists so that
+ # an extension middleware reading the body is bounded too, and so a
+ # declared oversize Content-Length is refused before any body is read.
+ app.add_middleware(RequestBodyLimitMiddleware)
+
# Delegates so the HTTP gate and WebSocketAuthMiddleware read a credential
# by exactly one rule; they guard the same token on two transports.
_extract_proxy_token = read_proxy_token
@@ -3895,20 +3937,6 @@ def create_app(config: ProxyConfig | None = None) -> FastAPI:
# handshake is refused before any project-prefix or routing work happens.
app.add_middleware(WebSocketAuthMiddleware, proxy_token=_proxy_token)
- # Third-party proxy extensions (Enterprise, custom plugins). Discovered via
- # the `headroom.proxy_extension` entry-point group, but **opt-in only**:
- # only names listed in config.proxy_extensions (CLI: --proxy-extension,
- # env: HEADROOM_PROXY_EXTENSIONS) actually get installed. Discovery alone
- # never runs third-party code. An extension that raises from its install()
- # disables *itself*: `install_all` logs it, prints a SKIPPED line to stderr
- # and the proxy keeps serving without that extension (extensions.py:169-183).
- # One bad plugin must not brick the proxy. Note the consequence: a plugin
- # whose licence gate raises is absent, not fatal — the feature fails closed,
- # the process does not.
- from headroom.proxy.extensions import install_all as _install_extensions
-
- _install_extensions(app, config, enabled=getattr(config, "proxy_extensions", None))
-
# Health & Metrics
@app.get("/livez")
async def livez():
diff --git a/tests/test_proxy_extension_ordering.py b/tests/test_proxy_extension_ordering.py
new file mode 100644
index 000000000..fb6d4b84c
--- /dev/null
+++ b/tests/test_proxy_extension_ordering.py
@@ -0,0 +1,444 @@
+"""Extension middleware runs inside the proxy's security gate and body ceiling.
+
+Regression tests for the ordering contract documented in
+``headroom/proxy/extensions.py``. Before this contract existed, ``install_all``
+ran after the gate was registered, and Starlette's prepend-on-register stack
+made every extension middleware the OUTERMOST layer: an extension that answered
+``POST /`` itself served unauthenticated callers, one that buffered the body
+held an unbounded body before the 401 was ever issued, and one that rewrote
+``Authorization`` did so before the gate compared it.
+
+The extension used here is a stand-in for the shapes the private extensions
+actually take (a middleware that claims a request and answers it, and one that
+buffers the whole body), installed through the real ``install_all`` seam by
+monkeypatching discovery, so the tests exercise the app exactly as
+``headroom proxy --proxy-extension <name>`` would.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+
+from headroom.cache.compression_store import reset_compression_store
+from headroom.proxy import extensions
+from headroom.proxy import helpers as proxy_helpers
+from headroom.proxy.request_body_limit import RequestBodyLimitMiddleware
+from headroom.proxy.server import ProxyConfig, WebSocketAuthMiddleware, create_app
+
+NONLOOPBACK = ("203.0.113.5", 44444) # TEST-NET-3, never loopback
+LOOPBACK = ("127.0.0.1", 12345)
+TOKEN = "s3cr3t-token"
+
+
+class _RecordingExtension:
+ """Raw ASGI middleware that records every scope it sees and answers ``POST /``.
+
+ Mirrors the shape of an extension that intercepts a vendor protocol on the
+ root path and relays it upstream itself, never calling the wrapped app.
+ """
+
+ seen: list[dict[str, Any]] = []
+
+ def __init__(self, app: Any) -> None:
+ self.app = app
+
+ async def __call__(self, scope: Any, receive: Any, send: Any) -> None:
+ if scope["type"] in {"http", "websocket"}:
+ type(self).seen.append({"type": scope["type"], "path": scope.get("path")})
+ if scope["type"] == "http" and scope.get("path") == "/" and scope.get("method") == "POST":
+ body = b'{"claimed_by":"extension"}'
+ await send(
+ {
+ "type": "http.response.start",
+ "status": 200,
+ "headers": [
+ (b"content-type", b"application/json"),
+ (b"content-length", str(len(body)).encode()),
+ ],
+ }
+ )
+ await send({"type": "http.response.body", "body": body})
+ return
+ await self.app(scope, receive, send)
+
+
+class _BufferingExtension:
+ """Raw ASGI middleware that buffers the full request body before passing it on.
+
+ Mirrors an extension that must read the JSON to decide how to route or
+ rewrite the request. ``max_seen`` records the largest body it ever held.
+ """
+
+ max_seen = 0
+
+ def __init__(self, app: Any) -> None:
+ self.app = app
+
+ async def __call__(self, scope: Any, receive: Any, send: Any) -> None:
+ if scope["type"] != "http":
+ await self.app(scope, receive, send)
+ return
+ chunks = bytearray()
+ while True:
+ message = await receive()
+ if message["type"] != "http.request":
+ break
+ chunks.extend(message.get("body") or b"")
+ type(self).max_seen = max(type(self).max_seen, len(chunks))
+ if not message.get("more_body"):
+ break
+ replay = {"type": "http.request", "body": bytes(chunks), "more_body": False}
+ sent = False
+
+ async def replay_receive() -> dict[str, Any]:
+ nonlocal sent
+ if not sent:
+ sent = True
+ return replay
+ return {"type": "http.disconnect"}
+
+ await self.app(scope, replay_receive, send)
+
+
+def _install_recording(app: Any, config: Any) -> None:
+ app.add_middleware(_RecordingExtension)
+
+
+def _install_buffering(app: Any, config: Any) -> None:
+ app.add_middleware(_BufferingExtension)
+
+
[email protected](autouse=True)
+def _reset_state() -> None:
+ _RecordingExtension.seen = []
+ _BufferingExtension.max_seen = 0
+
+
+def _make_app(monkeypatch, install, **overrides) -> Any:
+ reset_compression_store()
+ monkeypatch.setattr(extensions, "discover", lambda: iter([("test_ext", install)]))
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ proxy_extensions=["test_ext"],
+ **overrides,
+ )
+ return create_app(config)
+
+
+class TestExtensionMiddlewareIsInsideTheSecurityGate:
+ def test_unauthenticated_request_never_reaches_extension_middleware(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post("/", json={}, headers={"x-amz-target": "Vendor.Generate"})
+ assert resp.status_code == 401
+ assert resp.json() == {"error": "unauthorized"}
+ assert _RecordingExtension.seen == [], "extension ran before the proxy token was checked"
+
+ def test_authenticated_request_reaches_extension_middleware(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post("/", json={}, headers={"Authorization": f"Bearer {TOKEN}"})
+ assert resp.status_code == 200
+ assert resp.json() == {"claimed_by": "extension"}
+ assert [s["path"] for s in _RecordingExtension.seen] == ["/"]
+
+ def test_wrong_token_never_reaches_extension_middleware(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post("/", json={}, headers={"Authorization": "Bearer nope"})
+ assert resp.status_code == 401
+ assert _RecordingExtension.seen == []
+
+ def test_loopback_caller_is_exempt_and_reaches_extension(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://127.0.0.1", client=LOOPBACK) as c:
+ resp = c.post("/", json={})
+ assert resp.status_code == 200
+ assert [s["path"] for s in _RecordingExtension.seen] == ["/"]
+
+ def test_health_probe_stays_exempt_with_extension_installed(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.get("/health")
+ assert resp.status_code == 200
+ # Exempt from auth, so the extension legitimately sees it — the gate
+ # passes it through rather than the extension pre-empting the gate.
+ assert [s["path"] for s in _RecordingExtension.seen] == ["/health"]
+
+ def test_gate_security_headers_apply_to_extension_responses(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post("/", json={}, headers={"Authorization": f"Bearer {TOKEN}"})
+ assert resp.headers["X-Content-Type-Options"] == "nosniff"
+
+ def test_unauthenticated_websocket_never_reaches_extension_middleware(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ with pytest.raises(Exception): # noqa: B017 — TestClient surfaces the refused upgrade
+ with c.websocket_connect("/v1/responses"):
+ pass
+ assert _RecordingExtension.seen == [], "extension saw a refused WebSocket handshake"
+
+
+class TestBodyCeilingAppliesBeforeExtensionMiddleware:
+ def test_declared_oversize_body_is_refused_before_extension_buffers_it(self, monkeypatch):
+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
+ app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
+ payload = b"x" * 4096
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post(
+ "/v1/chat/completions",
+ content=payload,
+ headers={
+ "Authorization": f"Bearer {TOKEN}",
+ "content-type": "application/json",
+ },
+ )
+ assert resp.status_code == 413
+ assert resp.json()["error"]["code"] == "request_too_large"
+ assert _BufferingExtension.max_seen == 0
+
+ def test_chunked_oversize_body_is_cut_off_before_extension_holds_it_all(self, monkeypatch):
+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
+ app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
+ chunk = b"x" * 256
+
+ def body():
+ for _ in range(64): # 16 KiB, no Content-Length
+ yield chunk
+
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post(
+ "/v1/chat/completions",
+ content=body(),
+ headers={
+ "Authorization": f"Bearer {TOKEN}",
+ "content-type": "application/json",
+ },
+ )
+ assert resp.status_code == 413
+ # The ceiling plus at most one delivered message; never the whole 16 KiB.
+ # (TestClient delivers the generator as one message, so the extension
+ # sees nothing at all here; the multi-message case is covered by
+ # TestRequestBodyLimitMiddlewareDirectly below.)
+ assert _BufferingExtension.max_seen <= 1024 + len(chunk)
+
+ def test_anthropic_dialect_payload_on_messages_path(self, monkeypatch):
+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
+ app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post(
+ "/v1/messages",
+ content=b"x" * 4096,
+ headers={"Authorization": f"Bearer {TOKEN}", "content-type": "application/json"},
+ )
+ assert resp.status_code == 413
+ assert resp.json() == {
+ "type": "error",
+ "error": {
+ "type": "request_too_large",
+ "message": "Request body too large. Maximum size is 0MB",
+ },
+ }
+
+ def test_body_within_ceiling_reaches_extension_intact(self, monkeypatch):
+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
+ app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ # /stats is a local route, so the request completes without an upstream.
+ resp = c.post(
+ "/stats",
+ content=b"y" * 512,
+ headers={"Authorization": f"Bearer {TOKEN}"},
+ )
+ assert resp.status_code != 413
+ assert _BufferingExtension.max_seen == 512
+
+ def test_unauthenticated_oversize_body_is_refused_by_the_gate_first(self, monkeypatch):
+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
+ app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post("/v1/chat/completions", content=b"x" * 4096)
+ assert resp.status_code == 401
+ assert _BufferingExtension.max_seen == 0
+
+
+class TestStackOrderIsPinned:
+ """The contract is structural, so check the stack itself, not only its effects."""
+
+ @staticmethod
+ def _index_of(app, predicate) -> int:
+ for index, entry in enumerate(app.user_middleware):
+ if predicate(entry):
+ return index
+ raise AssertionError("middleware not found in stack")
+
+ def test_extension_middleware_is_inner_to_gate_and_body_limit(self, monkeypatch):
+ app = _make_app(monkeypatch, _install_recording, proxy_token=TOKEN)
+ added = app.state.extension_middleware
+ assert len(added) == 1
+ assert added[0].cls is _RecordingExtension
+
+ # Lower index == outer layer (Starlette prepends on add_middleware).
+ ext_index = self._index_of(app, lambda e: e.cls is _RecordingExtension)
+ limit_index = self._index_of(app, lambda e: e.cls is RequestBodyLimitMiddleware)
+ ws_auth_index = self._index_of(app, lambda e: e.cls is WebSocketAuthMiddleware)
+ gate_index = self._index_of(
+ app,
+ lambda e: getattr(e.kwargs.get("dispatch"), "__name__", "") == "_security_gate",
+ )
+ assert ws_auth_index < gate_index < limit_index < ext_index
+
+ def test_install_all_records_nothing_for_routes_only_extensions(self, monkeypatch):
+ def routes_only(app: Any, config: Any) -> None:
+ @app.get("/ext/ping")
+ async def ping():
+ return {"ok": True}
+
+ app = _make_app(monkeypatch, routes_only, proxy_token=TOKEN)
+ assert app.state.extension_middleware == []
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ assert c.get("/ext/ping").status_code == 401
+ assert c.get("/ext/ping", headers={"Authorization": f"Bearer {TOKEN}"}).json() == {
+ "ok": True
+ }
+
+
+class TestRequestBodyLimitMiddlewareDirectly:
+ """Drive the layer with a hand-rolled ASGI conversation to pin the metering."""
+
+ @staticmethod
+ def _http_scope(path="/v1/chat/completions", headers=()):
+ return {"type": "http", "method": "POST", "path": path, "headers": list(headers)}
+
+ @staticmethod
+ def _chunked_receive(chunks):
+ messages = [{"type": "http.request", "body": c, "more_body": True} for c in chunks]
+ messages[-1]["more_body"] = False
+ messages.append({"type": "http.disconnect"})
+ it = iter(messages)
+
+ async def receive():
+ return next(it)
+
+ return receive
+
+ async def test_stream_over_ceiling_answers_413_once_and_stops_the_reader(self):
+ held = bytearray()
+ downstream_sent: list[dict] = []
+ downstream_error: list[BaseException] = []
+
+ async def downstream(scope, receive, send):
+ try:
+ while True:
+ m = await receive()
+ if m["type"] != "http.request":
+ break
+ held.extend(m["body"])
+ if not m.get("more_body"):
+ break
+ except proxy_helpers.RequestBodyTooLarge as exc:
+ downstream_error.append(exc)
+ # A real handler that swallowed this would now try to answer;
+ # that answer must be discarded because the 413 already went out.
+ await send({"type": "http.response.start", "status": 400, "headers": []})
+ await send({"type": "http.response.body", "body": b"{}"})
+ downstream_sent.append({"attempted": True})
+ return
+ raise AssertionError("reader should have been stopped")
+
+ sent: list[dict] = []
+
+ async def send(m):
+ sent.append(m)
+
+ mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)
+ await mw(self._http_scope(), self._chunked_receive([b"a" * 400] * 5), send)
+
+ assert downstream_error, "the reader was not told to stop"
+ # The message that trips the ceiling is never delivered, so the reader
+ # holds at most the ceiling itself.
+ assert 0 < len(held) <= 1000
+ statuses = [m["status"] for m in sent if m["type"] == "http.response.start"]
+ assert statuses == [413] # exactly one refusal; the downstream 400 was dropped
+ assert downstream_sent == [{"attempted": True}]
+
+ async def test_body_under_ceiling_is_delivered_untouched(self):
+ held = bytearray()
+
+ async def downstream(scope, receive, send):
+ while True:
+ m = await receive()
+ held.extend(m["body"])
+ if not m.get("more_body"):
+ break
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+ await send({"type": "http.response.body", "body": b"ok"})
+
+ sent: list[dict] = []
+
+ async def send(m):
+ sent.append(m)
+
+ mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)
+ await mw(self._http_scope(), self._chunked_receive([b"a" * 300] * 3), send)
+ assert bytes(held) == b"a" * 900
+ assert [m["status"] for m in sent if m["type"] == "http.response.start"] == [200]
+
+ async def test_response_already_started_is_not_clobbered(self):
+ """A streaming handler that answered before finishing the body keeps its response."""
+
+ async def downstream(scope, receive, send):
+ await send({"type": "http.response.start", "status": 200, "headers": []})
+ await send({"type": "http.response.body", "body": b"partial", "more_body": True})
+ try:
+ while True:
+ m = await receive()
+ if not m.get("more_body"):
+ break
+ except proxy_helpers.RequestBodyTooLarge:
+ await send({"type": "http.response.body", "body": b"", "more_body": False})
+
+ sent: list[dict] = []
+
+ async def send(m):
+ sent.append(m)
+
+ mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)
+ await mw(self._http_scope(), self._chunked_receive([b"a" * 600] * 3), send)
+ assert [m["status"] for m in sent if m["type"] == "http.response.start"] == [200]
+ assert sent[-1] == {"type": "http.response.body", "body": b"", "more_body": False}
+
+ async def test_unparseable_content_length_falls_back_to_metering(self):
+ async def downstream(scope, receive, send):
+ while True:
+ m = await receive()
+ if not m.get("more_body"):
+ break
+
+ sent: list[dict] = []
+
+ async def send(m):
+ sent.append(m)
+
+ mw = RequestBodyLimitMiddleware(downstream, max_bytes=100)
+ scope = self._http_scope(headers=[(b"content-length", b"not-a-number")])
+ await mw(scope, self._chunked_receive([b"a" * 80, b"a" * 80]), send)
+ assert [m["status"] for m in sent if m["type"] == "http.response.start"] == [413]
+
+ async def test_non_http_scope_passes_through(self):
+ seen = []
+
+ async def downstream(scope, receive, send):
+ seen.append(scope["type"])
+
+ mw = RequestBodyLimitMiddleware(downstream, max_bytes=1)
+ await mw({"type": "websocket", "path": "/v1/live"}, None, None)
+ await mw({"type": "lifespan"}, None, None)
+ assert seen == ["websocket", "lifespan"]