Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: forward upstream ratelimit headers in streaming responses
jocel1bb1a576a7062
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex cbc597aa7..56b0d4719 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -42,6 +42,8 @@ if TYPE_CHECKING:from ..cache.compression_cache import CompressionCachefrom ..memory.tracker import ComponentStats, MemoryTracker+import contextlib+import httpxtry:@@ -4589,6 +4591,33 @@ class HeadroomProxy:and provider == "anthropic")+ # Open connection before generator to capture upstream response headers+ # (needed to forward ratelimit headers to the client via StreamingResponse)+ assert self.http_client is not None, "http_client must be initialized before streaming"+ try:+ _upstream_req = self.http_client.build_request("POST", url, json=body, headers=headers)+ upstream_response = await self.http_client.send(_upstream_req, stream=True)+ except (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) as e:+ error_msg = str(e)+ logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")++ async def _error_gen():+ error_event = {+ "type": "error",+ "error": {+ "type": "connection_error",+ "message": f"Failed to connect to upstream API: {error_msg}",+ },+ }+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n".encode()++ return StreamingResponse(_error_gen(), media_type="text/event-stream")++ # Forward upstream ratelimit headers to the client+ forwarded_headers = {+ k: v for k, v in upstream_response.headers.items() if "ratelimit" in k.lower()+ }+async def generate():nonlocal body, memory_enabled # May need to modify for continuation requests@@ -4597,9 +4626,7 @@ class HeadroomProxy:full_sse_data = ""try:- async with self.http_client.stream(- "POST", url, json=body, headers=headers- ) as response:+ async with contextlib.aclosing(upstream_response) as response:async for chunk in response.aiter_bytes():# Record TTFB on first chunkif stream_state["ttfb_ms"] is None:@@ -4794,6 +4821,7 @@ class HeadroomProxy:return StreamingResponse(generate(),media_type="text/event-stream",+ headers=forwarded_headers,)async def _stream_response_bedrock(