Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): preserve Bedrock body-limit error dialect (#3871)

charlieannaffc6edb4ba7a
diff --git a/headroom/proxy/request_body_limit.py b/headroom/proxy/request_body_limit.py
index 9a6a7a5c9..6985c384b 100644
--- a/headroom/proxy/request_body_limit.py
+++ b/headroom/proxy/request_body_limit.py
@@ -48,6 +48,21 @@ Send = Callable[[MutableMapping[str, Any]], Awaitable[None]]
# the OpenAI-style payload, which is what the handlers' own 413s use.
_ANTHROPIC_PATH_MARKERS = ("/v1/messages",)
+# The AWS Bedrock InvokeModel passthrough speaks a third dialect: its handler
+# (``handle_bedrock_invoke``) answers a too-large body with
+# ``{"error": {"type": "request_too_large", "message": ...}}``. Its routes are
+# ``/model/{model_id:path}/invoke`` and
+# ``/model/{model_id:path}/invoke-with-response-stream``
+# (``headroom/providers/proxy_routes.py``), and model ids may contain slashes
+# (ARN-style inference profiles), so the match is a suffix check under the
+# ``/model/`` prefix rather than a fixed-path table.
+_BEDROCK_INVOKE_PREFIX = "/model/"
+_BEDROCK_INVOKE_SUFFIXES = ("/invoke", "/invoke-with-response-stream")
+
+
+def _is_bedrock_invoke_path(path: str) -> bool:
+ return path.startswith(_BEDROCK_INVOKE_PREFIX) and path.endswith(_BEDROCK_INVOKE_SUFFIXES)
+
def _too_large_payload(path: str, limit: int) -> bytes:
"""Mirror the handlers' own 413 payloads so clients see one shape per dialect."""
@@ -57,6 +72,13 @@ def _too_large_payload(path: str, limit: int) -> bytes:
"type": "error",
"error": {"type": "request_too_large", "message": message},
}
+ elif _is_bedrock_invoke_path(path):
+ payload = {
+ "error": {
+ "type": "request_too_large",
+ "message": message,
+ }
+ }
else:
payload = {
"error": {
diff --git a/tests/test_proxy/test_bedrock_passthrough.py b/tests/test_proxy/test_bedrock_passthrough.py
index 694930131..45d4ed029 100644
--- a/tests/test_proxy/test_bedrock_passthrough.py
+++ b/tests/test_proxy/test_bedrock_passthrough.py
@@ -352,6 +352,81 @@ def test_oversized_body_returns_413_and_never_forwards(monkeypatch):
http.send.assert_not_called()
+ "route", [INVOKE, INVOKE_STREAM], ids=["invoke", "invoke-with-response-stream"]
+)
+def test_body_ceiling_413_uses_bedrock_handler_dialect(monkeypatch, route):
+ """The body ceiling is enforced outside the handler, but its 413 must keep
+ the route's own wire dialect: exactly the payload ``handle_bedrock_invoke``
+ returns for ``RequestBodyTooLarge`` —
+ ``{"error": {"type": "request_too_large", "message": ...}}`` — with the
+ request refused before any upstream call.
+ """
+ import headroom.proxy.helpers as helpers_mod
+
+ small_cap = 1024
+ monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)
+
+ app = create_app(_make_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ http = _install_fake_client(proxy, _FakeUpstream())
+ proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))
+ body = {"messages": [{"role": "user", "content": "x" * (small_cap * 4)}], "max_tokens": 8}
+ resp = client.post(route, json=body)
+
+ assert resp.status_code == 413
+ assert resp.json() == {
+ "error": {
+ "type": "request_too_large",
+ "message": f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB",
+ }
+ }
+ http.build_request.assert_not_called()
+ http.send.assert_not_called()
+
+
+def test_body_ceiling_413_leaves_other_dialects_unchanged(monkeypatch):
+ """The dialect fix is scoped to the Bedrock InvokeModel routes: ``/v1/messages``
+ keeps the Anthropic envelope and OpenAI chat keeps
+ ``{"error": {"type": "invalid_request_error", "code": "request_too_large"}}``.
+ """
+ import headroom.proxy.helpers as helpers_mod
+
+ small_cap = 1024
+ monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)
+
+ app = create_app(_make_config())
+ with TestClient(app) as client:
+ proxy = client.app.state.proxy
+ http = _install_fake_client(proxy, _FakeUpstream())
+ proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))
+ oversized = b"x" * (small_cap * 4)
+ messages = client.post(
+ "/v1/messages",
+ content=oversized,
+ headers={"content-type": "application/json"},
+ )
+ chat = client.post(
+ "/v1/chat/completions",
+ content=oversized,
+ headers={"content-type": "application/json"},
+ )
+
+ message = f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB"
+ assert messages.status_code == 413
+ assert messages.json() == {
+ "type": "error",
+ "error": {"type": "request_too_large", "message": message},
+ }
+ assert chat.status_code == 413
+ assert chat.json() == {
+ "error": {"message": message, "type": "invalid_request_error", "code": "request_too_large"}
+ }
+ http.build_request.assert_not_called()
+ http.send.assert_not_called()
+
+
# ── metrics ───────────────────────────────────────────────────────────
diff --git a/tests/test_proxy_extension_ordering.py b/tests/test_proxy_extension_ordering.py
index fb6d4b84c..cee4128c5 100644
--- a/tests/test_proxy_extension_ordering.py
+++ b/tests/test_proxy_extension_ordering.py
@@ -17,6 +17,7 @@ monkeypatching discovery, so the tests exercise the app exactly as
from __future__ import annotations
+import json
from typing import Any
import pytest
@@ -247,6 +248,42 @@ class TestBodyCeilingAppliesBeforeExtensionMiddleware:
},
}
+ @pytest.mark.parametrize(
+ "route",
+ [
+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",
+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",
+ ],
+ ids=["invoke", "invoke-with-response-stream"],
+ )
+ def test_bedrock_invoke_dialect_payload_on_model_paths(self, monkeypatch, route):
+ """The two Bedrock InvokeModel routes keep handle_bedrock_invoke's wire
+ dialect on the body ceiling: exactly
+ ``{"error": {"type": "request_too_large", "message": ...}}`` — not the
+ OpenAI-style payload — and the extension never buffers the body.
+ """
+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
+ app = _make_app(
+ monkeypatch,
+ _install_buffering,
+ proxy_token=TOKEN,
+ bedrock_api_url="http://127.0.0.1:4000",
+ )
+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:
+ resp = c.post(
+ route,
+ content=b"x" * 4096,
+ headers={"Authorization": f"Bearer {TOKEN}", "content-type": "application/json"},
+ )
+ assert resp.status_code == 413
+ assert resp.json() == {
+ "error": {
+ "type": "request_too_large",
+ "message": "Request body too large. Maximum size is 0MB",
+ }
+ }
+ assert _BufferingExtension.max_seen == 0
+
def test_body_within_ceiling_reaches_extension_intact(self, monkeypatch):
monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)
app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)
@@ -369,6 +406,68 @@ class TestRequestBodyLimitMiddlewareDirectly:
assert statuses == [413] # exactly one refusal; the downstream 400 was dropped
assert downstream_sent == [{"attempted": True}]
+ @pytest.mark.parametrize(
+ "path",
+ [
+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",
+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",
+ ],
+ ids=["invoke", "invoke-with-response-stream"],
+ )
+ async def test_stream_trip_answers_once_in_bedrock_dialect(self, path):
+ """On a metered-stream trip over a Bedrock InvokeModel route the single
+ refusal carries handle_bedrock_invoke's dialect, and the handler's own
+ 413 answer (its RequestBodyTooLarge response) is discarded by the
+ response-once state: exactly one response reaches the wire.
+ """
+ downstream_sent: list[dict] = []
+
+ async def downstream(scope, receive, send):
+ try:
+ while True:
+ m = await receive()
+ if m["type"] != "http.request":
+ break
+ if not m.get("more_body"):
+ break
+ except proxy_helpers.RequestBodyTooLarge:
+ # handle_bedrock_invoke answers RequestBodyTooLarge with its own
+ # 413; response-once must drop it because the ceiling already answered.
+ body = b'{"error": {"type": "request_too_large", "message": "from handler"}}'
+ await send(
+ {
+ "type": "http.response.start",
+ "status": 413,
+ "headers": [
+ (b"content-type", b"application/json"),
+ (b"content-length", str(len(body)).encode()),
+ ],
+ }
+ )
+ await send({"type": "http.response.body", "body": body})
+ downstream_sent.append({"attempted": True})
+ return
+ raise AssertionError("reader should have been stopped")
+
+ sent: list[dict] = []
+
+ async def send(m):
+ sent.append(m)
+
+ mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)
+ await mw(self._http_scope(path=path), self._chunked_receive([b"a" * 400] * 5), send)
+
+ statuses = [m["status"] for m in sent if m["type"] == "http.response.start"]
+ assert statuses == [413] # exactly one response; the handler's 413 was dropped
+ body = b"".join(m.get("body") or b"" for m in sent if m["type"] == "http.response.body")
+ assert json.loads(body) == {
+ "error": {
+ "type": "request_too_large",
+ "message": "Request body too large. Maximum size is 0MB",
+ }
+ }
+ assert downstream_sent == [{"attempted": True}]
+
async def test_body_under_ceiling_is_delivered_untouched(self):
held = bytearray()