Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): stop the 0.39.0 TPM limiter from refusing large-context requests forever (#3806)
gglucass7968122658c3
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdxindex a9fa4cf33..eddee0d8d 100644--- a/docs/content/docs/proxy.mdx+++ b/docs/content/docs/proxy.mdx@@ -274,7 +274,7 @@ Coding agents re-read the same files repeatedly; these control how stale reads a| `--connect-timeout-seconds` | `10` | Upstream connect timeout (seconds). || `--retry-max-attempts` | `3` | Upstream retries on transient failure. || `--limit-concurrency` | `1000` | Concurrent connections before returning 503. |-| `--rpm` / `--tpm` | `60` / `100000` | Requests- and tokens-per-minute rate limits (disable with `--no-rate-limit`). |+| `--rpm` / `--tpm` | `60` / unlimited | Requests- and tokens-per-minute rate limits (disable with `--no-rate-limit`). || `--budget` / `--budget-period` | none / `daily` | Spend cap in USD per period; over-budget requests get 429. || `--workers` / `HEADROOM_WORKERS` | `1` | Uvicorn worker processes. |diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 58a25b239..15da1f71c 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -372,7 +372,7 @@ def dashboard(port: int, no_open: bool) -> None:default=None,type=click.IntRange(min=1),envvar="HEADROOM_TPM",- help="Max tokens per minute. Env: HEADROOM_TPM. Default: 100000.",+ help="Max tokens per minute. Env: HEADROOM_TPM. Default: unlimited.",)@click.option("--no-ccr",@@ -1324,7 +1324,7 @@ def proxy(cache_enabled=not no_cache,rate_limit_enabled=not no_rate_limit,rate_limit_requests_per_minute=rpm if rpm is not None else 60,- rate_limit_tokens_per_minute=tpm if tpm is not None else 100_000,+ rate_limit_tokens_per_minute=tpm,compress_user_messages=_get_env_bool("HEADROOM_COMPRESS_USER_MESSAGES", False),periodic_malloc_trim_enabled=_get_env_bool("HEADROOM_MALLOC_TRIM", default_periodic_malloc_trim()diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 5957f5773..0f87c89eb 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -329,7 +329,10 @@ class ProxyConfig:# Rate limitingrate_limit_enabled: bool = Truerate_limit_requests_per_minute: int = 60- rate_limit_tokens_per_minute: int = 100000+ # None = no token limit. Before 0.39.0 the token bucket was never consumed,+ # so an implicit default here throttled every large-context agent session+ # the moment #3350 started enforcing it. Opt in with --tpm / HEADROOM_TPM.+ rate_limit_tokens_per_minute: int | None = None# Retryretry_enabled: bool = Truediff --git a/headroom/proxy/rate_limit_policy.py b/headroom/proxy/rate_limit_policy.pyindex c3525cac2..9e57686c6 100644--- a/headroom/proxy/rate_limit_policy.py+++ b/headroom/proxy/rate_limit_policy.py@@ -22,11 +22,19 @@ def consume_from_bucket(requested_tokens: float,rate_per_minute: float,) -> tuple[bool, float, float]:- """Return ``(allowed, remaining_tokens, wait_seconds)`` for a token request."""- if available_tokens >= requested_tokens:+ """Return ``(allowed, remaining_tokens, wait_seconds)`` for a token request.++ The bucket never holds more than ``rate_per_minute``, so a request larger+ than that could never be admitted and its ``wait_seconds`` would never come+ true. Such a request waits for a full bucket instead, then is charged in+ full: the balance goes negative and later requests wait it off, so the+ configured average rate still holds.+ """+ required = min(requested_tokens, rate_per_minute)+ if available_tokens >= required:return True, available_tokens - requested_tokens, 0.0- wait_seconds = (requested_tokens - available_tokens) * (60.0 / rate_per_minute)+ wait_seconds = (required - available_tokens) * (60.0 / rate_per_minute)return False, available_tokens, wait_secondsdiff --git a/headroom/proxy/rate_limiter.py b/headroom/proxy/rate_limiter.pyindex abfb40f2e..5aaae622d 100644--- a/headroom/proxy/rate_limiter.py+++ b/headroom/proxy/rate_limiter.py@@ -24,7 +24,7 @@ class TokenBucketRateLimiter:def __init__(self,requests_per_minute: int = 60,- tokens_per_minute: int = 100000,+ tokens_per_minute: int | None = None,):self.requests_per_minute = requests_per_minuteself.tokens_per_minute = tokens_per_minute@@ -55,10 +55,10 @@ class TokenBucketRateLimiter:self._request_buckets[key] = statereturn state- def _token_bucket(self, key: str) -> RateLimitState:+ def _token_bucket(self, key: str, capacity: int) -> RateLimitState:state = self._token_buckets.get(key)if state is None:- state = RateLimitState(tokens=self.tokens_per_minute, last_update=time.time())+ state = RateLimitState(tokens=capacity, last_update=time.time())self._token_buckets[key] = statereturn state@@ -89,16 +89,19 @@ class TokenBucketRateLimiter:return allowed, wait_secondsasync def check_tokens(self, key: str, token_count: int) -> tuple[bool, float]:- """Check if token usage is allowed."""+ """Check if token usage is allowed. ``tokens_per_minute=None`` never limits."""+ tpm = self.tokens_per_minute+ if tpm is None:+ return True, 0.0async with self._lock:self._touch_bucket(key)- state = self._token_bucket(key)- available = self._refill(state, self.tokens_per_minute)+ state = self._token_bucket(key, tpm)+ available = self._refill(state, tpm)allowed, state.tokens, wait_seconds = consume_from_bucket(available_tokens=available,requested_tokens=token_count,- rate_per_minute=self.tokens_per_minute,+ rate_per_minute=tpm,)return allowed, wait_secondsdiff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex d8a0ba236..7ffe7a89d 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -5912,7 +5912,7 @@ def run_server(║ FEATURES: ║║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║-║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║+║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {f"{config.rate_limit_tokens_per_minute:,}" if config.rate_limit_tokens_per_minute else "unlimited"} tok/min) ║║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║║ Code-Aware: {code_aware_status:<52}║@@ -6398,7 +6398,9 @@ if __name__ == "__main__":# Rate limitingparser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")- parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")+ parser.add_argument(+ "--tpm", type=int, default=None, help="Tokens per minute (default: unlimited)"+ )# Costparser.add_argument("--budget", type=float, help="Budget limit in USD")diff --git a/tests/test_cli_proxy_env.py b/tests/test_cli_proxy_env.pyindex f4b8df63d..5a7d74538 100644--- a/tests/test_cli_proxy_env.py+++ b/tests/test_cli_proxy_env.py@@ -1337,7 +1337,7 @@ class TestCLIProxyRpmTpm:assert captured_config["config"].rate_limit_requests_per_minute == 20def test_tpm_default(self, runner):- """Without --tpm, rate_limit_tokens_per_minute defaults to 100000."""+ """Without --tpm, there is no token limit (the pre-0.39.0 behaviour)."""captured_config = {}def mock_run_server(config, **kwargs):@@ -1347,7 +1347,7 @@ class TestCLIProxyRpmTpm:result = runner.invoke(main, ["proxy"], catch_exceptions=False)assert result.exit_code == 0, result.output- assert captured_config["config"].rate_limit_tokens_per_minute == 100000+ assert captured_config["config"].rate_limit_tokens_per_minute is Nonedef test_tpm_flag(self, runner):"""--tpm 50000 should set rate_limit_tokens_per_minute to 50000."""diff --git a/tests/test_proxy_tpm_enforcement.py b/tests/test_proxy_tpm_enforcement.pyindex f16586c94..3c1b8b663 100644--- a/tests/test_proxy_tpm_enforcement.py+++ b/tests/test_proxy_tpm_enforcement.py@@ -14,6 +14,9 @@ from headroom.proxy.server import ProxyConfig, create_app # noqa: E402def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) -> None:+ # The first request finds a full bucket and is admitted even though it is+ # larger than the whole bucket (otherwise it could never pass); it is+ # charged in full, so the second one is refused before upstream.monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")config = ProxyConfig(optimize=False,@@ -29,26 +32,32 @@ def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) ->image_optimize=False,)- with TestClient(create_app(config)) as client:+ with TestClient(create_app(config), raise_server_exceptions=False) as client:proxy = client.app.state.proxytokenizer = MagicMock()proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))- proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))-- response = client.post(- "/v1/messages",- headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},- json={- "model": "claude-haiku-4-5",- "max_tokens": 64,- "messages": [{"role": "user", "content": "large request"}],- "stream": False,- },- )+ proxy._retry_request = AsyncMock(side_effect=RuntimeError("upstream stub"))++ def post():+ return client.post(+ "/v1/messages",+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-haiku-4-5",+ "max_tokens": 64,+ "messages": [{"role": "user", "content": "large request"}],+ "stream": False,+ },+ )++ assert post().status_code != 429+ assert proxy._retry_request.await_count == 1++ response = post()assert response.status_code == 429assert response.json()["detail"].startswith("Token rate limited.")- proxy._retry_request.assert_not_awaited()+ assert proxy._retry_request.await_count == 1assert len(proxy.rate_limiter._token_buckets) == 1@@ -100,15 +109,20 @@ def test_other_handlers_reject_request_over_tpm_before_upstream(image_optimize=False,)- with TestClient(create_app(config)) as client:+ with TestClient(create_app(config), raise_server_exceptions=False) as client:proxy = client.app.state.proxytokenizer = MagicMock()proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))- proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))+ proxy._retry_request = AsyncMock(side_effect=RuntimeError("upstream stub"))++ assert client.post(path, headers=headers, json=body).status_code != 429+ (bucket,) = proxy.rate_limiter._token_buckets.values()+ assert bucket.tokens < 0 # admitted and charged in full+ upstream_calls = proxy._retry_request.await_countresponse = client.post(path, headers=headers, json=body)assert response.status_code == 429assert response.json()["detail"].startswith("Token rate limited.")- proxy._retry_request.assert_not_awaited()+ assert proxy._retry_request.await_count == upstream_callsassert len(proxy.rate_limiter._token_buckets) == 1diff --git a/tests/test_rate_limit_policy.py b/tests/test_rate_limit_policy.pyindex cf504d788..089f30028 100644--- a/tests/test_rate_limit_policy.py+++ b/tests/test_rate_limit_policy.py@@ -57,6 +57,46 @@ def test_consume_from_bucket_denies_and_reports_wait_time() -> None:assert wait_seconds == 0.5+def test_consume_from_bucket_admits_request_larger_than_bucket_once_full() -> None:+ # 0.39.0 enforced TPM with a 100k default; a 232k-token coding-agent turn+ # was refused forever with a Retry-After that never came true.+ allowed, remaining, wait_seconds = consume_from_bucket(+ available_tokens=100_000,+ requested_tokens=232_000,+ rate_per_minute=100_000,+ )++ assert allowed is True+ assert remaining == -132_000+ assert wait_seconds == 0+++def test_consume_from_bucket_oversized_request_waits_only_for_a_full_bucket() -> None:+ allowed, remaining, wait_seconds = consume_from_bucket(+ available_tokens=40_000,+ requested_tokens=232_000,+ rate_per_minute=100_000,+ )++ assert allowed is False+ assert remaining == 40_000+ assert wait_seconds == 36+++def test_consume_from_bucket_debt_delays_the_next_request() -> None:+ # Charged in full, so the average rate holds: after the 232k request the+ # next small one waits out the 132k debt plus its own tokens.+ allowed, remaining, wait_seconds = consume_from_bucket(+ available_tokens=-132_000,+ requested_tokens=1_000,+ rate_per_minute=100_000,+ )++ assert allowed is False+ assert remaining == -132_000+ assert wait_seconds == 79.8++def test_stale_bucket_keys_returns_only_old_buckets() -> None:assert stale_bucket_keys({"fresh": 950, "edge": 400, "stale": 399},diff --git a/tests/test_rate_limiter.py b/tests/test_rate_limiter.pyindex e6d19a6d7..f1ea3d11c 100644--- a/tests/test_rate_limiter.py+++ b/tests/test_rate_limiter.py@@ -104,3 +104,13 @@ async def test_request_and_token_limits_still_debit_independently() -> None:assert (await limiter.check_request("client"))[0] is Falseassert (await limiter.check_tokens("client", 5))[0] is Trueassert (await limiter.check_tokens("client", 1))[0] is False+++async def test_token_limit_is_off_unless_configured() -> None:+ # 0.39.0 started enforcing an implicit 100k default and 429'd every+ # large-context agent turn; the default is now no token limit.+ limiter = TokenBucketRateLimiter()++ assert await limiter.check_tokens("client", 900_000) == (True, 0.0)+ assert not limiter._token_buckets