Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(perf): make rate limiter bucket checks constant-time (#3373)
C0sm0catsa8bd9becb300
diff --git a/headroom/proxy/rate_limiter.py b/headroom/proxy/rate_limiter.pyindex 50100d823..abfb40f2e 100644--- a/headroom/proxy/rate_limiter.py+++ b/headroom/proxy/rate_limiter.py@@ -8,14 +8,11 @@ Extracted from server.py for maintainability.from __future__ import annotationsimport asyncio-import loggingimport time-from collections import defaultdict+from collections import OrderedDictfrom headroom.proxy.models import RateLimitState-from headroom.proxy.rate_limit_policy import consume_from_bucket, refilled_tokens, stale_bucket_keys--logger = logging.getLogger("headroom.proxy")+from headroom.proxy.rate_limit_policy import consume_from_bucket, refilled_tokens# Maximum rate limiter buckets (prevents DoS via spoofed API keys)MAX_RATE_LIMITER_BUCKETS = 1000@@ -32,28 +29,38 @@ class TokenBucketRateLimiter:self.requests_per_minute = requests_per_minuteself.tokens_per_minute = tokens_per_minute- # Per-key buckets (key = API key or IP)- self._request_buckets: dict[str, RateLimitState] = defaultdict(- lambda: RateLimitState(tokens=requests_per_minute, last_update=time.time())- )- self._token_buckets: dict[str, RateLimitState] = defaultdict(- lambda: RateLimitState(tokens=tokens_per_minute, last_update=time.time())- )+ # Per-key buckets (key = API key or IP). A shared LRU keeps request and+ # token state on the same bounded lifecycle without scanning all keys.+ self._request_buckets: dict[str, RateLimitState] = {}+ self._token_buckets: dict[str, RateLimitState] = {}+ self._bucket_lru: OrderedDict[str, None] = OrderedDict()self._lock = asyncio.Lock()- async def _cleanup_stale_buckets(self) -> None:- """Remove buckets that haven't been used in the last 10 minutes."""- now = time.time()- stale_keys = stale_bucket_keys(- {k: v.last_update for k, v in self._request_buckets.items()},- now=now,- stale_after_seconds=600,- )- for k in stale_keys:- del self._request_buckets[k]- self._token_buckets.pop(k, None)- if stale_keys:- logger.debug(f"Cleaned up {len(stale_keys)} stale rate limiter buckets")+ def _touch_bucket(self, key: str) -> None:+ """Mark a bucket active, evicting the least-recently-used key at capacity."""+ if key in self._bucket_lru:+ self._bucket_lru.move_to_end(key)+ return++ if len(self._bucket_lru) >= MAX_RATE_LIMITER_BUCKETS:+ evicted_key, _ = self._bucket_lru.popitem(last=False)+ self._request_buckets.pop(evicted_key, None)+ self._token_buckets.pop(evicted_key, None)+ self._bucket_lru[key] = None++ def _request_bucket(self, key: str) -> RateLimitState:+ state = self._request_buckets.get(key)+ if state is None:+ state = RateLimitState(tokens=self.requests_per_minute, last_update=time.time())+ self._request_buckets[key] = state+ return state++ def _token_bucket(self, key: str) -> RateLimitState:+ state = self._token_buckets.get(key)+ if state is None:+ state = RateLimitState(tokens=self.tokens_per_minute, last_update=time.time())+ self._token_buckets[key] = state+ return statedef _refill(self, state: RateLimitState, rate_per_minute: float) -> float:"""Refill bucket based on elapsed time."""@@ -70,10 +77,8 @@ class TokenBucketRateLimiter:async def check_request(self, key: str = "default") -> tuple[bool, float]:"""Check if request is allowed. Returns (allowed, wait_seconds)."""async with self._lock:- # Prevent unbounded bucket growth from spoofed keys- if len(self._request_buckets) > MAX_RATE_LIMITER_BUCKETS:- await self._cleanup_stale_buckets()- state = self._request_buckets[key]+ self._touch_bucket(key)+ state = self._request_bucket(key)available = self._refill(state, self.requests_per_minute)allowed, state.tokens, wait_seconds = consume_from_bucket(@@ -86,7 +91,8 @@ class TokenBucketRateLimiter:async def check_tokens(self, key: str, token_count: int) -> tuple[bool, float]:"""Check if token usage is allowed."""async with self._lock:- state = self._token_buckets[key]+ self._touch_bucket(key)+ state = self._token_bucket(key)available = self._refill(state, self.tokens_per_minute)allowed, state.tokens, wait_seconds = consume_from_bucket(@@ -102,5 +108,5 @@ class TokenBucketRateLimiter:return {"requests_per_minute": self.requests_per_minute,"tokens_per_minute": self.tokens_per_minute,- "active_keys": len(self._request_buckets),+ "active_keys": len(self._bucket_lru),}diff --git a/tests/test_rate_limiter.py b/tests/test_rate_limiter.pynew file mode 100644index 000000000..e6d19a6d7--- /dev/null+++ b/tests/test_rate_limiter.py@@ -0,0 +1,106 @@+"""Regression tests for the async token-bucket rate limiter."""++from __future__ import annotations++from collections import OrderedDict++import pytest++from headroom.proxy.models import RateLimitState+from headroom.proxy.rate_limiter import MAX_RATE_LIMITER_BUCKETS, TokenBucketRateLimiter+++class _NoIterationDict(dict[str, RateLimitState]):+ """Bucket table that fails if a hot-path operation scans every state."""++ def __iter__(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket states")++ def items(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket states")++ def keys(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket states")++ def values(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket states")+++class _NoIterationOrderedDict(OrderedDict[str, None]):+ """OrderedDict that fails if a hot-path operation scans all identities."""++ def __iter__(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket identities")++ def items(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket identities")++ def keys(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket identities")++ def values(self): # type: ignore[override]+ raise AssertionError("rate-limit hot path scanned all bucket identities")+++async def test_existing_bucket_check_does_not_scan_all_identities() -> None:+ limiter = TokenBucketRateLimiter(requests_per_minute=1_000_000)+ for index in range(MAX_RATE_LIMITER_BUCKETS):+ await limiter.check_request(f"client-{index}")++ limiter._request_buckets = _NoIterationDict(limiter._request_buckets)+ limiter._bucket_lru = _NoIterationOrderedDict(limiter._bucket_lru)++ allowed, wait_seconds = await limiter.check_request("client-0")++ assert allowed is True+ assert wait_seconds == 0+++async def test_bucket_state_is_hard_capped_and_lru_evicted() -> None:+ limiter = TokenBucketRateLimiter(+ requests_per_minute=1_000_000,+ tokens_per_minute=1_000_000,+ )+ for index in range(MAX_RATE_LIMITER_BUCKETS):+ key = f"client-{index}"+ await limiter.check_request(key)+ await limiter.check_tokens(key, 1)++ # Keep client-0 recent so client-1 is now the least-recently-used identity.+ await limiter.check_request("client-0")+ await limiter.check_request("overflow-client")++ assert len(limiter._bucket_lru) == MAX_RATE_LIMITER_BUCKETS+ assert len(limiter._request_buckets) == MAX_RATE_LIMITER_BUCKETS+ assert len(limiter._token_buckets) == MAX_RATE_LIMITER_BUCKETS - 1+ assert "client-0" in limiter._request_buckets+ assert "client-1" not in limiter._request_buckets+ assert "client-1" not in limiter._token_buckets+ assert "overflow-client" in limiter._request_buckets+++async def test_token_only_checks_share_the_same_hard_cap() -> None:+ limiter = TokenBucketRateLimiter(tokens_per_minute=1_000_000)++ for index in range(MAX_RATE_LIMITER_BUCKETS + 50):+ allowed, wait_seconds = await limiter.check_tokens(f"client-{index}", 1)+ assert allowed is True+ assert wait_seconds == 0++ assert len(limiter._bucket_lru) == MAX_RATE_LIMITER_BUCKETS+ assert len(limiter._token_buckets) == MAX_RATE_LIMITER_BUCKETS+ assert not limiter._request_buckets+ assert (await limiter.stats())["active_keys"] == MAX_RATE_LIMITER_BUCKETS+++async def test_request_and_token_limits_still_debit_independently() -> None:+ limiter = TokenBucketRateLimiter(requests_per_minute=1, tokens_per_minute=5)++ assert (await limiter.check_request("client"))[0] is True+ assert (await limiter.check_request("client"))[0] is False+ assert (await limiter.check_tokens("client", 5))[0] is True+ assert (await limiter.check_tokens("client", 1))[0] is False