Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): preserve Bedrock body-limit error dialect (#3871)
charlieannaffc6edb4ba7a
diff --git a/headroom/proxy/request_body_limit.py b/headroom/proxy/request_body_limit.pyindex 9a6a7a5c9..6985c384b 100644--- a/headroom/proxy/request_body_limit.py+++ b/headroom/proxy/request_body_limit.py@@ -48,6 +48,21 @@ Send = Callable[[MutableMapping[str, Any]], Awaitable[None]]# the OpenAI-style payload, which is what the handlers' own 413s use._ANTHROPIC_PATH_MARKERS = ("/v1/messages",)+# The AWS Bedrock InvokeModel passthrough speaks a third dialect: its handler+# (``handle_bedrock_invoke``) answers a too-large body with+# ``{"error": {"type": "request_too_large", "message": ...}}``. Its routes are+# ``/model/{model_id:path}/invoke`` and+# ``/model/{model_id:path}/invoke-with-response-stream``+# (``headroom/providers/proxy_routes.py``), and model ids may contain slashes+# (ARN-style inference profiles), so the match is a suffix check under the+# ``/model/`` prefix rather than a fixed-path table.+_BEDROCK_INVOKE_PREFIX = "/model/"+_BEDROCK_INVOKE_SUFFIXES = ("/invoke", "/invoke-with-response-stream")+++def _is_bedrock_invoke_path(path: str) -> bool:+ return path.startswith(_BEDROCK_INVOKE_PREFIX) and path.endswith(_BEDROCK_INVOKE_SUFFIXES)+def _too_large_payload(path: str, limit: int) -> bytes:"""Mirror the handlers' own 413 payloads so clients see one shape per dialect."""@@ -57,6 +72,13 @@ def _too_large_payload(path: str, limit: int) -> bytes:"type": "error","error": {"type": "request_too_large", "message": message},}+ elif _is_bedrock_invoke_path(path):+ payload = {+ "error": {+ "type": "request_too_large",+ "message": message,+ }+ }else:payload = {"error": {diff --git a/tests/test_proxy/test_bedrock_passthrough.py b/tests/test_proxy/test_bedrock_passthrough.pyindex 694930131..45d4ed029 100644--- a/tests/test_proxy/test_bedrock_passthrough.py+++ b/tests/test_proxy/test_bedrock_passthrough.py@@ -352,6 +352,81 @@ def test_oversized_body_returns_413_and_never_forwards(monkeypatch):http.send.assert_not_called()+ "route", [INVOKE, INVOKE_STREAM], ids=["invoke", "invoke-with-response-stream"]+)+def test_body_ceiling_413_uses_bedrock_handler_dialect(monkeypatch, route):+ """The body ceiling is enforced outside the handler, but its 413 must keep+ the route's own wire dialect: exactly the payload ``handle_bedrock_invoke``+ returns for ``RequestBodyTooLarge`` —+ ``{"error": {"type": "request_too_large", "message": ...}}`` — with the+ request refused before any upstream call.+ """+ import headroom.proxy.helpers as helpers_mod++ small_cap = 1024+ monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)++ app = create_app(_make_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ http = _install_fake_client(proxy, _FakeUpstream())+ proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))+ body = {"messages": [{"role": "user", "content": "x" * (small_cap * 4)}], "max_tokens": 8}+ resp = client.post(route, json=body)++ assert resp.status_code == 413+ assert resp.json() == {+ "error": {+ "type": "request_too_large",+ "message": f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB",+ }+ }+ http.build_request.assert_not_called()+ http.send.assert_not_called()+++def test_body_ceiling_413_leaves_other_dialects_unchanged(monkeypatch):+ """The dialect fix is scoped to the Bedrock InvokeModel routes: ``/v1/messages``+ keeps the Anthropic envelope and OpenAI chat keeps+ ``{"error": {"type": "invalid_request_error", "code": "request_too_large"}}``.+ """+ import headroom.proxy.helpers as helpers_mod++ small_cap = 1024+ monkeypatch.setattr(helpers_mod, "MAX_REQUEST_BODY_SIZE", small_cap)++ app = create_app(_make_config())+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ http = _install_fake_client(proxy, _FakeUpstream())+ proxy.anthropic_pipeline.apply = MagicMock(side_effect=AssertionError("should not run"))+ oversized = b"x" * (small_cap * 4)+ messages = client.post(+ "/v1/messages",+ content=oversized,+ headers={"content-type": "application/json"},+ )+ chat = client.post(+ "/v1/chat/completions",+ content=oversized,+ headers={"content-type": "application/json"},+ )++ message = f"Request body too large. Maximum size is {small_cap // (1024 * 1024)}MB"+ assert messages.status_code == 413+ assert messages.json() == {+ "type": "error",+ "error": {"type": "request_too_large", "message": message},+ }+ assert chat.status_code == 413+ assert chat.json() == {+ "error": {"message": message, "type": "invalid_request_error", "code": "request_too_large"}+ }+ http.build_request.assert_not_called()+ http.send.assert_not_called()++# ── metrics ───────────────────────────────────────────────────────────diff --git a/tests/test_proxy_extension_ordering.py b/tests/test_proxy_extension_ordering.pyindex fb6d4b84c..cee4128c5 100644--- a/tests/test_proxy_extension_ordering.py+++ b/tests/test_proxy_extension_ordering.py@@ -17,6 +17,7 @@ monkeypatching discovery, so the tests exercise the app exactly asfrom __future__ import annotations+import jsonfrom typing import Anyimport pytest@@ -247,6 +248,42 @@ class TestBodyCeilingAppliesBeforeExtensionMiddleware:},}+ @pytest.mark.parametrize(+ "route",+ [+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",+ ],+ ids=["invoke", "invoke-with-response-stream"],+ )+ def test_bedrock_invoke_dialect_payload_on_model_paths(self, monkeypatch, route):+ """The two Bedrock InvokeModel routes keep handle_bedrock_invoke's wire+ dialect on the body ceiling: exactly+ ``{"error": {"type": "request_too_large", "message": ...}}`` — not the+ OpenAI-style payload — and the extension never buffers the body.+ """+ monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)+ app = _make_app(+ monkeypatch,+ _install_buffering,+ proxy_token=TOKEN,+ bedrock_api_url="http://127.0.0.1:4000",+ )+ with TestClient(app, base_url="http://testserver", client=NONLOOPBACK) as c:+ resp = c.post(+ route,+ content=b"x" * 4096,+ headers={"Authorization": f"Bearer {TOKEN}", "content-type": "application/json"},+ )+ assert resp.status_code == 413+ assert resp.json() == {+ "error": {+ "type": "request_too_large",+ "message": "Request body too large. Maximum size is 0MB",+ }+ }+ assert _BufferingExtension.max_seen == 0+def test_body_within_ceiling_reaches_extension_intact(self, monkeypatch):monkeypatch.setattr(proxy_helpers, "MAX_REQUEST_BODY_SIZE", 1024)app = _make_app(monkeypatch, _install_buffering, proxy_token=TOKEN)@@ -369,6 +406,68 @@ class TestRequestBodyLimitMiddlewareDirectly:assert statuses == [413] # exactly one refusal; the downstream 400 was droppedassert downstream_sent == [{"attempted": True}]+ @pytest.mark.parametrize(+ "path",+ [+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke",+ "/model/anthropic.claude-3-5-sonnet-20241022-v2:0/invoke-with-response-stream",+ ],+ ids=["invoke", "invoke-with-response-stream"],+ )+ async def test_stream_trip_answers_once_in_bedrock_dialect(self, path):+ """On a metered-stream trip over a Bedrock InvokeModel route the single+ refusal carries handle_bedrock_invoke's dialect, and the handler's own+ 413 answer (its RequestBodyTooLarge response) is discarded by the+ response-once state: exactly one response reaches the wire.+ """+ downstream_sent: list[dict] = []++ async def downstream(scope, receive, send):+ try:+ while True:+ m = await receive()+ if m["type"] != "http.request":+ break+ if not m.get("more_body"):+ break+ except proxy_helpers.RequestBodyTooLarge:+ # handle_bedrock_invoke answers RequestBodyTooLarge with its own+ # 413; response-once must drop it because the ceiling already answered.+ body = b'{"error": {"type": "request_too_large", "message": "from handler"}}'+ await send(+ {+ "type": "http.response.start",+ "status": 413,+ "headers": [+ (b"content-type", b"application/json"),+ (b"content-length", str(len(body)).encode()),+ ],+ }+ )+ await send({"type": "http.response.body", "body": body})+ downstream_sent.append({"attempted": True})+ return+ raise AssertionError("reader should have been stopped")++ sent: list[dict] = []++ async def send(m):+ sent.append(m)++ mw = RequestBodyLimitMiddleware(downstream, max_bytes=1000)+ await mw(self._http_scope(path=path), self._chunked_receive([b"a" * 400] * 5), send)++ statuses = [m["status"] for m in sent if m["type"] == "http.response.start"]+ assert statuses == [413] # exactly one response; the handler's 413 was dropped+ body = b"".join(m.get("body") or b"" for m in sent if m["type"] == "http.response.body")+ assert json.loads(body) == {+ "error": {+ "type": "request_too_large",+ "message": "Request body too large. Maximum size is 0MB",+ }+ }+ assert downstream_sent == [{"attempted": True}]+async def test_body_under_ceiling_is_delivered_untouched(self):held = bytearray()