Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): stop the 0.39.0 TPM limiter from refusing large-context requests forever (#3806)

gglucass7968122658c3
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index a9fa4cf33..eddee0d8d 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -274,7 +274,7 @@ Coding agents re-read the same files repeatedly; these control how stale reads a
| `--connect-timeout-seconds` | `10` | Upstream connect timeout (seconds). |
| `--retry-max-attempts` | `3` | Upstream retries on transient failure. |
| `--limit-concurrency` | `1000` | Concurrent connections before returning 503. |
-| `--rpm` / `--tpm` | `60` / `100000` | Requests- and tokens-per-minute rate limits (disable with `--no-rate-limit`). |
+| `--rpm` / `--tpm` | `60` / unlimited | Requests- and tokens-per-minute rate limits (disable with `--no-rate-limit`). |
| `--budget` / `--budget-period` | none / `daily` | Spend cap in USD per period; over-budget requests get 429. |
| `--workers` / `HEADROOM_WORKERS` | `1` | Uvicorn worker processes. |
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index 58a25b239..15da1f71c 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -372,7 +372,7 @@ def dashboard(port: int, no_open: bool) -> None:
default=None,
type=click.IntRange(min=1),
envvar="HEADROOM_TPM",
- help="Max tokens per minute. Env: HEADROOM_TPM. Default: 100000.",
+ help="Max tokens per minute. Env: HEADROOM_TPM. Default: unlimited.",
)
@click.option(
"--no-ccr",
@@ -1324,7 +1324,7 @@ def proxy(
cache_enabled=not no_cache,
rate_limit_enabled=not no_rate_limit,
rate_limit_requests_per_minute=rpm if rpm is not None else 60,
- rate_limit_tokens_per_minute=tpm if tpm is not None else 100_000,
+ rate_limit_tokens_per_minute=tpm,
compress_user_messages=_get_env_bool("HEADROOM_COMPRESS_USER_MESSAGES", False),
periodic_malloc_trim_enabled=_get_env_bool(
"HEADROOM_MALLOC_TRIM", default_periodic_malloc_trim()
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index 5957f5773..0f87c89eb 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -329,7 +329,10 @@ class ProxyConfig:
# Rate limiting
rate_limit_enabled: bool = True
rate_limit_requests_per_minute: int = 60
- rate_limit_tokens_per_minute: int = 100000
+ # None = no token limit. Before 0.39.0 the token bucket was never consumed,
+ # so an implicit default here throttled every large-context agent session
+ # the moment #3350 started enforcing it. Opt in with --tpm / HEADROOM_TPM.
+ rate_limit_tokens_per_minute: int | None = None
# Retry
retry_enabled: bool = True
diff --git a/headroom/proxy/rate_limit_policy.py b/headroom/proxy/rate_limit_policy.py
index c3525cac2..9e57686c6 100644
--- a/headroom/proxy/rate_limit_policy.py
+++ b/headroom/proxy/rate_limit_policy.py
@@ -22,11 +22,19 @@ def consume_from_bucket(
requested_tokens: float,
rate_per_minute: float,
) -> tuple[bool, float, float]:
- """Return ``(allowed, remaining_tokens, wait_seconds)`` for a token request."""
- if available_tokens >= requested_tokens:
+ """Return ``(allowed, remaining_tokens, wait_seconds)`` for a token request.
+
+ The bucket never holds more than ``rate_per_minute``, so a request larger
+ than that could never be admitted and its ``wait_seconds`` would never come
+ true. Such a request waits for a full bucket instead, then is charged in
+ full: the balance goes negative and later requests wait it off, so the
+ configured average rate still holds.
+ """
+ required = min(requested_tokens, rate_per_minute)
+ if available_tokens >= required:
return True, available_tokens - requested_tokens, 0.0
- wait_seconds = (requested_tokens - available_tokens) * (60.0 / rate_per_minute)
+ wait_seconds = (required - available_tokens) * (60.0 / rate_per_minute)
return False, available_tokens, wait_seconds
diff --git a/headroom/proxy/rate_limiter.py b/headroom/proxy/rate_limiter.py
index abfb40f2e..5aaae622d 100644
--- a/headroom/proxy/rate_limiter.py
+++ b/headroom/proxy/rate_limiter.py
@@ -24,7 +24,7 @@ class TokenBucketRateLimiter:
def __init__(
self,
requests_per_minute: int = 60,
- tokens_per_minute: int = 100000,
+ tokens_per_minute: int | None = None,
):
self.requests_per_minute = requests_per_minute
self.tokens_per_minute = tokens_per_minute
@@ -55,10 +55,10 @@ class TokenBucketRateLimiter:
self._request_buckets[key] = state
return state
- def _token_bucket(self, key: str) -> RateLimitState:
+ def _token_bucket(self, key: str, capacity: int) -> RateLimitState:
state = self._token_buckets.get(key)
if state is None:
- state = RateLimitState(tokens=self.tokens_per_minute, last_update=time.time())
+ state = RateLimitState(tokens=capacity, last_update=time.time())
self._token_buckets[key] = state
return state
@@ -89,16 +89,19 @@ class TokenBucketRateLimiter:
return allowed, wait_seconds
async def check_tokens(self, key: str, token_count: int) -> tuple[bool, float]:
- """Check if token usage is allowed."""
+ """Check if token usage is allowed. ``tokens_per_minute=None`` never limits."""
+ tpm = self.tokens_per_minute
+ if tpm is None:
+ return True, 0.0
async with self._lock:
self._touch_bucket(key)
- state = self._token_bucket(key)
- available = self._refill(state, self.tokens_per_minute)
+ state = self._token_bucket(key, tpm)
+ available = self._refill(state, tpm)
allowed, state.tokens, wait_seconds = consume_from_bucket(
available_tokens=available,
requested_tokens=token_count,
- rate_per_minute=self.tokens_per_minute,
+ rate_per_minute=tpm,
)
return allowed, wait_seconds
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index d8a0ba236..7ffe7a89d 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -5912,7 +5912,7 @@ def run_server(
║ FEATURES: ║
║ Optimization: {"ENABLED " if config.optimize else "DISABLED"} ║
║ Caching: {"ENABLED " if config.cache_enabled else "DISABLED"} (TTL: {config.cache_ttl_seconds}s) ║
-║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {config.rate_limit_tokens_per_minute:,} tok/min) ║
+║ Rate Limiting: {"ENABLED " if config.rate_limit_enabled else "DISABLED"} ({config.rate_limit_requests_per_minute} req/min, {f"{config.rate_limit_tokens_per_minute:,}" if config.rate_limit_tokens_per_minute else "unlimited"} tok/min) ║
║ Retry: {"ENABLED " if config.retry_enabled else "DISABLED"} (max {config.retry_max_attempts} attempts) ║
║ Cost Tracking: {"ENABLED " if config.cost_tracking_enabled else "DISABLED"} (budget: {"$" + str(config.budget_limit_usd) + "/" + config.budget_period if config.budget_limit_usd else "unlimited"}) ║
║ Code-Aware: {code_aware_status:<52}║
@@ -6398,7 +6398,9 @@ if __name__ == "__main__":
# Rate limiting
parser.add_argument("--no-rate-limit", action="store_true", help="Disable rate limiting")
parser.add_argument("--rpm", type=int, default=60, help="Requests per minute")
- parser.add_argument("--tpm", type=int, default=100000, help="Tokens per minute")
+ parser.add_argument(
+ "--tpm", type=int, default=None, help="Tokens per minute (default: unlimited)"
+ )
# Cost
parser.add_argument("--budget", type=float, help="Budget limit in USD")
diff --git a/tests/test_cli_proxy_env.py b/tests/test_cli_proxy_env.py
index f4b8df63d..5a7d74538 100644
--- a/tests/test_cli_proxy_env.py
+++ b/tests/test_cli_proxy_env.py
@@ -1337,7 +1337,7 @@ class TestCLIProxyRpmTpm:
assert captured_config["config"].rate_limit_requests_per_minute == 20
def test_tpm_default(self, runner):
- """Without --tpm, rate_limit_tokens_per_minute defaults to 100000."""
+ """Without --tpm, there is no token limit (the pre-0.39.0 behaviour)."""
captured_config = {}
def mock_run_server(config, **kwargs):
@@ -1347,7 +1347,7 @@ class TestCLIProxyRpmTpm:
result = runner.invoke(main, ["proxy"], catch_exceptions=False)
assert result.exit_code == 0, result.output
- assert captured_config["config"].rate_limit_tokens_per_minute == 100000
+ assert captured_config["config"].rate_limit_tokens_per_minute is None
def test_tpm_flag(self, runner):
"""--tpm 50000 should set rate_limit_tokens_per_minute to 50000."""
diff --git a/tests/test_proxy_tpm_enforcement.py b/tests/test_proxy_tpm_enforcement.py
index f16586c94..3c1b8b663 100644
--- a/tests/test_proxy_tpm_enforcement.py
+++ b/tests/test_proxy_tpm_enforcement.py
@@ -14,6 +14,9 @@ from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) -> None:
+ # The first request finds a full bucket and is admitted even though it is
+ # larger than the whole bucket (otherwise it could never pass); it is
+ # charged in full, so the second one is refused before upstream.
monkeypatch.setenv("HEADROOM_SKIP_UPSTREAM_CHECK", "1")
config = ProxyConfig(
optimize=False,
@@ -29,26 +32,32 @@ def test_anthropic_request_over_tpm_is_rejected_before_upstream(monkeypatch) ->
image_optimize=False,
)
- with TestClient(create_app(config)) as client:
+ with TestClient(create_app(config), raise_server_exceptions=False) as client:
proxy = client.app.state.proxy
tokenizer = MagicMock()
proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
- proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
-
- response = client.post(
- "/v1/messages",
- headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
- json={
- "model": "claude-haiku-4-5",
- "max_tokens": 64,
- "messages": [{"role": "user", "content": "large request"}],
- "stream": False,
- },
- )
+ proxy._retry_request = AsyncMock(side_effect=RuntimeError("upstream stub"))
+
+ def post():
+ return client.post(
+ "/v1/messages",
+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-haiku-4-5",
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "large request"}],
+ "stream": False,
+ },
+ )
+
+ assert post().status_code != 429
+ assert proxy._retry_request.await_count == 1
+
+ response = post()
assert response.status_code == 429
assert response.json()["detail"].startswith("Token rate limited.")
- proxy._retry_request.assert_not_awaited()
+ assert proxy._retry_request.await_count == 1
assert len(proxy.rate_limiter._token_buckets) == 1
@@ -100,15 +109,20 @@ def test_other_handlers_reject_request_over_tpm_before_upstream(
image_optimize=False,
)
- with TestClient(create_app(config)) as client:
+ with TestClient(create_app(config), raise_server_exceptions=False) as client:
proxy = client.app.state.proxy
tokenizer = MagicMock()
proxy._count_tokens_offloaded = AsyncMock(return_value=(tokenizer, 5_000))
- proxy._retry_request = AsyncMock(side_effect=AssertionError("upstream must not be called"))
+ proxy._retry_request = AsyncMock(side_effect=RuntimeError("upstream stub"))
+
+ assert client.post(path, headers=headers, json=body).status_code != 429
+ (bucket,) = proxy.rate_limiter._token_buckets.values()
+ assert bucket.tokens < 0 # admitted and charged in full
+ upstream_calls = proxy._retry_request.await_count
response = client.post(path, headers=headers, json=body)
assert response.status_code == 429
assert response.json()["detail"].startswith("Token rate limited.")
- proxy._retry_request.assert_not_awaited()
+ assert proxy._retry_request.await_count == upstream_calls
assert len(proxy.rate_limiter._token_buckets) == 1
diff --git a/tests/test_rate_limit_policy.py b/tests/test_rate_limit_policy.py
index cf504d788..089f30028 100644
--- a/tests/test_rate_limit_policy.py
+++ b/tests/test_rate_limit_policy.py
@@ -57,6 +57,46 @@ def test_consume_from_bucket_denies_and_reports_wait_time() -> None:
assert wait_seconds == 0.5
+def test_consume_from_bucket_admits_request_larger_than_bucket_once_full() -> None:
+ # 0.39.0 enforced TPM with a 100k default; a 232k-token coding-agent turn
+ # was refused forever with a Retry-After that never came true.
+ allowed, remaining, wait_seconds = consume_from_bucket(
+ available_tokens=100_000,
+ requested_tokens=232_000,
+ rate_per_minute=100_000,
+ )
+
+ assert allowed is True
+ assert remaining == -132_000
+ assert wait_seconds == 0
+
+
+def test_consume_from_bucket_oversized_request_waits_only_for_a_full_bucket() -> None:
+ allowed, remaining, wait_seconds = consume_from_bucket(
+ available_tokens=40_000,
+ requested_tokens=232_000,
+ rate_per_minute=100_000,
+ )
+
+ assert allowed is False
+ assert remaining == 40_000
+ assert wait_seconds == 36
+
+
+def test_consume_from_bucket_debt_delays_the_next_request() -> None:
+ # Charged in full, so the average rate holds: after the 232k request the
+ # next small one waits out the 132k debt plus its own tokens.
+ allowed, remaining, wait_seconds = consume_from_bucket(
+ available_tokens=-132_000,
+ requested_tokens=1_000,
+ rate_per_minute=100_000,
+ )
+
+ assert allowed is False
+ assert remaining == -132_000
+ assert wait_seconds == 79.8
+
+
def test_stale_bucket_keys_returns_only_old_buckets() -> None:
assert stale_bucket_keys(
{"fresh": 950, "edge": 400, "stale": 399},
diff --git a/tests/test_rate_limiter.py b/tests/test_rate_limiter.py
index e6d19a6d7..f1ea3d11c 100644
--- a/tests/test_rate_limiter.py
+++ b/tests/test_rate_limiter.py
@@ -104,3 +104,13 @@ async def test_request_and_token_limits_still_debit_independently() -> None:
assert (await limiter.check_request("client"))[0] is False
assert (await limiter.check_tokens("client", 5))[0] is True
assert (await limiter.check_tokens("client", 1))[0] is False
+
+
+async def test_token_limit_is_off_unless_configured() -> None:
+ # 0.39.0 started enforcing an implicit 100k default and 429'd every
+ # large-context agent turn; the default is now no token limit.
+ limiter = TokenBucketRateLimiter()
+
+ assert await limiter.check_tokens("client", 900_000) == (True, 0.0)
+ assert not limiter._token_buckets