Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Make provider backoff admission monotonic (#1053)

Ali Khokhar23ed6bc87ef4
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 3df5d329..041891bb 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -474,7 +474,12 @@ owns provider-ID-to-constructor selection; concrete provider classes own their
transport inheritance or composition. Each lazy provider receives a fresh
`ProviderRateLimiter`; there is no process singleton or second limiter registry.
The provider cache already guarantees one provider and limiter per provider ID
-within a generation. Retired generations
+within a generation. Provider admission combines a strict proactive window with
+one reactive backoff deadline. Positive backoffs can only extend that deadline,
+and admission loops until proactive capacity and the final reactive check are
+simultaneously available. The proactive timestamp is recorded only when that
+check succeeds, so a concurrent 429/5xx cannot be missed, shortened, consume
+unused quota, or release queued requests as an expiry burst. Retired generations
retain their own synchronization state until request leases drain, while new
generations and separate server instances never reuse it. Hot replacement
therefore begins with fresh quota state; an old and new generation enforce
diff --git a/pyproject.toml b/pyproject.toml
index 9d841243..79337ae0 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "free-claude-code"
-version = "3.5.0"
+version = "3.5.1"
description = "Middleware between Claude Code CLI (Anthropic API) and NVIDIA NIM"
readme = "README.md"
requires-python = ">=3.14.0"
diff --git a/src/free_claude_code/core/rate_limit.py b/src/free_claude_code/core/rate_limit.py
index b60b491a..15bd9f4f 100644
--- a/src/free_claude_code/core/rate_limit.py
+++ b/src/free_claude_code/core/rate_limit.py
@@ -3,6 +3,7 @@
import asyncio
import time
from collections import deque
+from collections.abc import Callable
class StrictSlidingWindowLimiter:
@@ -29,6 +30,17 @@ class StrictSlidingWindowLimiter:
self._lock = asyncio.Lock()
async def acquire(self) -> None:
+ await self._acquire(None)
+
+ async def acquire_if(self, allowed: Callable[[], bool]) -> bool:
+ """Record an acquisition only if ``allowed`` still holds at admission.
+
+ Capacity is awaited first. The synchronous condition and timestamp write
+ then run without yielding, so a rejected admission consumes no quota.
+ """
+ return await self._acquire(allowed)
+
+ async def _acquire(self, allowed: Callable[[], bool] | None) -> bool:
while True:
wait_time = 0.0
async with self._lock:
@@ -39,8 +51,10 @@ class StrictSlidingWindowLimiter:
self._times.popleft()
if len(self._times) < self._rate_limit:
- self._times.append(now)
- return
+ if allowed is not None and not allowed():
+ return False
+ self._times.append(time.monotonic())
+ return True
oldest = self._times[0]
wait_time = max(0.0, (oldest + self._rate_window) - now)
diff --git a/src/free_claude_code/providers/rate_limit.py b/src/free_claude_code/providers/rate_limit.py
index 4e90a71a..b2efb6e4 100644
--- a/src/free_claude_code/providers/rate_limit.py
+++ b/src/free_claude_code/providers/rate_limit.py
@@ -71,39 +71,43 @@ class ProviderRateLimiter:
Returns:
True if was reactively blocked and waited, False otherwise.
"""
- # 1. Reactive check: Wait if someone hit a reactive backoff (429/5xx retries)
+ # A reactive deadline can be installed or extended while this task waits
+ # for proactive capacity. Commit the proactive timestamp only if that
+ # deadline is still clear, so retries neither burst nor consume unused quota.
waited_reactively = False
- now = time.monotonic()
- if now < self._blocked_until:
- wait_time = self._blocked_until - now
+ while True:
+ waited_reactively = (
+ await self._wait_for_reactive_block() or waited_reactively
+ )
+ if await self._proactive_limiter.acquire_if(lambda: not self.is_blocked()):
+ return waited_reactively
+
+ async def _wait_for_reactive_block(self) -> bool:
+ waited = False
+ while (wait_time := self.remaining_wait()) > 0:
logger.warning(
- f"Provider rate limit active (reactive), waiting {wait_time:.1f}s..."
+ "Provider rate limit active (reactive), waiting {:.1f}s...",
+ wait_time,
)
await asyncio.sleep(wait_time)
- waited_reactively = True
-
- # 2. Proactive check: strict rolling window (no bursts beyond N in last W seconds)
- await self._acquire_proactive_slot()
- return waited_reactively
-
- async def _acquire_proactive_slot(self) -> None:
- """
- Acquire a proactive slot enforcing a strict rolling window.
-
- Guarantees: at most `self._rate_limit` acquisitions in any interval of length
- `self._rate_window` (seconds).
- """
- await self._proactive_limiter.acquire()
+ waited = True
+ return waited
- def set_blocked(self, seconds: float = 60) -> None:
+ def extend_reactive_block(self, seconds: float) -> None:
"""
- Set this provider's block for the specified seconds (reactive).
+ Extend this provider's reactive block by at least ``seconds`` from now.
Args:
- seconds: How long to block (default 60s)
+ seconds: Positive minimum duration for the resulting block.
"""
- self._blocked_until = time.monotonic() + seconds
- logger.warning(f"Provider rate limit set for {seconds:.1f}s (reactive)")
+ if seconds <= 0:
+ raise ValueError("reactive block duration must be > 0")
+ now = time.monotonic()
+ self._blocked_until = max(self._blocked_until, now + seconds)
+ logger.warning(
+ "Provider rate limit set for {:.1f}s (reactive)",
+ max(0.0, self._blocked_until - now),
+ )
def is_blocked(self) -> bool:
"""Check if currently reactively blocked."""
@@ -211,7 +215,7 @@ class ProviderRateLimiter:
delay_s=round(delay, 3),
)
if status is not None:
- self.set_blocked(delay)
+ self.extend_reactive_block(delay)
await asyncio.sleep(delay)
assert last_exc is not None
diff --git a/src/free_claude_code/providers/transports/anthropic_messages/stream.py b/src/free_claude_code/providers/transports/anthropic_messages/stream.py
index 23b943e4..b3605bcb 100644
--- a/src/free_claude_code/providers/transports/anthropic_messages/stream.py
+++ b/src/free_claude_code/providers/transports/anthropic_messages/stream.py
@@ -227,7 +227,9 @@ class AnthropicMessagesStreamAdapter:
provider_name=tag,
read_timeout_s=self._transport._config.http_read_timeout,
request_id=self._request_id,
- mark_rate_limited=self._transport._rate_limiter.set_blocked,
+ mark_rate_limited=(
+ self._transport._rate_limiter.extend_reactive_block
+ ),
)
error_trace: dict[str, Any] = {
diff --git a/src/free_claude_code/providers/transports/openai_chat/stream.py b/src/free_claude_code/providers/transports/openai_chat/stream.py
index 16f8aaea..6f9b1471 100644
--- a/src/free_claude_code/providers/transports/openai_chat/stream.py
+++ b/src/free_claude_code/providers/transports/openai_chat/stream.py
@@ -280,7 +280,9 @@ class OpenAIChatStreamAdapter:
provider_name=tag,
read_timeout_s=self._transport._config.http_read_timeout,
request_id=self._request_id,
- mark_rate_limited=self._transport._rate_limiter.set_blocked,
+ mark_rate_limited=(
+ self._transport._rate_limiter.extend_reactive_block
+ ),
)
error_trace: dict[str, Any] = {
"stage": "provider",
diff --git a/tests/core/test_strict_sliding_window.py b/tests/core/test_strict_sliding_window.py
index 130d6491..7622ba2f 100644
--- a/tests/core/test_strict_sliding_window.py
+++ b/tests/core/test_strict_sliding_window.py
@@ -1,9 +1,11 @@
"""Direct tests for :class:`core.rate_limit.StrictSlidingWindowLimiter`."""
+import asyncio
import time
import pytest
+import free_claude_code.core.rate_limit as rate_limit_module
from free_claude_code.core.rate_limit import StrictSlidingWindowLimiter
@@ -31,6 +33,42 @@ async def test_strict_window_async_context_manager():
assert time.monotonic() - start >= 0.1
+async def test_rejected_conditional_acquisition_does_not_consume_capacity():
+ lim = StrictSlidingWindowLimiter(rate_limit=1, rate_window=60)
+
+ assert await lim.acquire_if(lambda: False) is False
+
+ await asyncio.wait_for(lim.acquire(), timeout=0.1)
+
+
+async def test_conditional_acquisition_records_predicate_commit_time(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ now = 0.0
+ sleep_delays: list[float] = []
+ lim = StrictSlidingWindowLimiter(rate_limit=1, rate_window=10)
+
+ def advance_during_condition() -> bool:
+ nonlocal now
+ now = 100.0
+ return True
+
+ async def advance_during_sleep(delay: float) -> None:
+ nonlocal now
+ sleep_delays.append(delay)
+ now += delay
+
+ monkeypatch.setattr(rate_limit_module.time, "monotonic", lambda: now)
+ monkeypatch.setattr(rate_limit_module.asyncio, "sleep", advance_during_sleep)
+
+ assert await lim.acquire_if(advance_during_condition) is True
+ await lim.acquire()
+
+ assert sleep_delays == [10.0]
+
+
def test_strict_window_rejects_invalid_config():
with pytest.raises(ValueError):
StrictSlidingWindowLimiter(rate_limit=0, rate_window=1.0)
diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.py
index f59ecf5b..61c6cc46 100644
--- a/tests/providers/test_provider_rate_limit.py
+++ b/tests/providers/test_provider_rate_limit.py
@@ -7,6 +7,7 @@ import openai
import pytest
from httpx import Request
+import free_claude_code.providers.rate_limit as rate_limit_module
from free_claude_code.providers.failure_policy import (
retryable_upstream_status,
retryable_upstream_transport_error,
@@ -114,7 +115,7 @@ class TestProviderRateLimiter:
@pytest.mark.asyncio
async def test_reactive_blocking(self):
"""
- Test reactive blocking when set_blocked is called.
+ Test reactive blocking when a deadline is extended.
Logic ported from verify_provider_limiter.py
"""
limiter = ProviderRateLimiter()
@@ -123,7 +124,7 @@ class TestProviderRateLimiter:
# Manually block for 1.5s
block_time = 1.5
- limiter.set_blocked(block_time)
+ limiter.extend_reactive_block(block_time)
assert limiter.is_blocked()
@@ -143,15 +144,141 @@ class TestProviderRateLimiter:
)
@pytest.mark.asyncio
- async def test_set_blocked_zero_immediately_unblocks(self):
- """set_blocked(0) should not actually block."""
+ async def test_reactive_block_at_proactive_commit_retries_without_reserving(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
- limiter.set_blocked(0)
+ proactive_attempts = 0
+ proactive_reservations = 0
+ reactive_checks = 0
+
+ async def acquire_if_allowed(_allowed) -> bool:
+ nonlocal proactive_attempts, proactive_reservations
+ proactive_attempts += 1
+ if proactive_attempts == 1:
+ return False
+ proactive_reservations += 1
+ return True
+
+ async def wait_reactively() -> bool:
+ nonlocal reactive_checks
+ reactive_checks += 1
+ return reactive_checks == 2
+
+ monkeypatch.setattr(
+ limiter._proactive_limiter,
+ "acquire_if",
+ acquire_if_allowed,
+ )
+ monkeypatch.setattr(
+ limiter,
+ "_wait_for_reactive_block",
+ wait_reactively,
+ )
+
+ assert await limiter.wait_if_blocked() is True
+ assert proactive_attempts == 2
+ assert proactive_reservations == 1
+ assert reactive_checks == 2
+
+ @pytest.mark.asyncio
+ async def test_reactive_wait_rechecks_an_extended_deadline(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ now = 0.0
+ sleep_delays: list[float] = []
+ limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
+
+ async def advance_time(delay: float) -> None:
+ nonlocal now
+ sleep_delays.append(delay)
+ if len(sleep_delays) == 1:
+ now = 5.0
+ limiter.extend_reactive_block(10.0)
+ now = 10.0
+ return
+ now += delay
+
+ monkeypatch.setattr(rate_limit_module.time, "monotonic", lambda: now)
+ monkeypatch.setattr(rate_limit_module.asyncio, "sleep", advance_time)
+ monkeypatch.setattr(
+ limiter._proactive_limiter,
+ "acquire_if",
+ AsyncMock(return_value=True),
+ )
+ limiter.extend_reactive_block(10.0)
- # Should not be blocked since 0 seconds from now is already past
- await asyncio.sleep(0.01)
+ assert await limiter.wait_if_blocked() is True
+ assert sleep_delays == [10.0, 5.0]
assert limiter.is_blocked() is False
- assert limiter.remaining_wait() == 0
+
+ @pytest.mark.asyncio
+ async def test_reactive_wait_propagates_cancellation(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
+ sleep_started = asyncio.Event()
+
+ async def wait_forever(_delay: float) -> None:
+ sleep_started.set()
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(rate_limit_module.asyncio, "sleep", wait_forever)
+ limiter.extend_reactive_block(60.0)
+ waiter = asyncio.create_task(limiter.wait_if_blocked())
+ await sleep_started.wait()
+
+ waiter.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+ assert limiter.is_blocked() is True
+
+ @pytest.mark.asyncio
+ async def test_proactive_wait_propagates_cancellation(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
+ acquire_started = asyncio.Event()
+
+ async def wait_forever(_allowed) -> bool:
+ acquire_started.set()
+ await asyncio.Event().wait()
+ return True
+
+ monkeypatch.setattr(limiter._proactive_limiter, "acquire_if", wait_forever)
+ waiter = asyncio.create_task(limiter.wait_if_blocked())
+ await acquire_started.wait()
+
+ waiter.cancel()
+
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+
+ def test_shorter_reactive_backoff_does_not_shorten_existing_deadline(
+ self,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ now = 100.0
+ monkeypatch.setattr(rate_limit_module.time, "monotonic", lambda: now)
+ limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
+
+ limiter.extend_reactive_block(10.0)
+ now = 102.0
+ limiter.extend_reactive_block(1.0)
+
+ assert limiter.remaining_wait() == 8.0
+
+ @pytest.mark.parametrize("seconds", [0.0, -1.0])
+ def test_reactive_backoff_duration_must_be_positive(self, seconds: float) -> None:
+ limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
+
+ with pytest.raises(ValueError, match="reactive block duration must be > 0"):
+ limiter.extend_reactive_block(seconds)
@pytest.mark.asyncio
async def test_remaining_wait_when_not_blocked(self):
@@ -163,7 +290,7 @@ class TestProviderRateLimiter:
async def test_remaining_wait_decreases(self):
"""remaining_wait() should decrease over time."""
limiter = ProviderRateLimiter(rate_limit=100, rate_window=60)
- limiter.set_blocked(2.0)
+ limiter.extend_reactive_block(2.0)
wait1 = limiter.remaining_wait()
assert wait1 > 1.5
@@ -197,7 +324,7 @@ class TestProviderRateLimiter:
limiter1 = ProviderRateLimiter(rate_limit=10, rate_window=1)
limiter2 = ProviderRateLimiter(rate_limit=10, rate_window=1)
- limiter1.set_blocked(1)
+ limiter1.extend_reactive_block(1)
assert limiter1 is not limiter2
assert limiter1.is_blocked() is True
@@ -432,7 +559,7 @@ class TestProviderRateLimiter:
return "ok"
with (
- patch.object(limiter, "set_blocked") as set_blocked,
+ patch.object(limiter, "extend_reactive_block") as extend_block,
patch("asyncio.sleep", new_callable=AsyncMock) as sleep,
):
result = await limiter.execute_with_retry(
@@ -445,7 +572,7 @@ class TestProviderRateLimiter:
assert result == "ok"
assert call_count == 2
- set_blocked.assert_not_called()
+ extend_block.assert_not_called()
sleep.assert_awaited_once()
@pytest.mark.asyncio
@@ -463,7 +590,7 @@ class TestProviderRateLimiter:
return "ok"
with (
- patch.object(limiter, "set_blocked") as set_blocked,
+ patch.object(limiter, "extend_reactive_block") as extend_block,
patch("asyncio.sleep", new_callable=AsyncMock) as sleep,
):
result = await limiter.execute_with_retry(
@@ -476,7 +603,7 @@ class TestProviderRateLimiter:
assert result == "ok"
assert call_count == 2
- set_blocked.assert_not_called()
+ extend_block.assert_not_called()
sleep.assert_awaited_once()
@pytest.mark.asyncio
@@ -492,7 +619,7 @@ class TestProviderRateLimiter:
raise httpx.ConnectError("connect failed")
with (
- patch.object(limiter, "set_blocked") as set_blocked,
+ patch.object(limiter, "extend_reactive_block") as extend_block,
patch("asyncio.sleep", new_callable=AsyncMock) as sleep,
pytest.raises(httpx.ConnectError),
):
@@ -504,7 +631,7 @@ class TestProviderRateLimiter:
)
assert call_count == UPSTREAM_TRANSIENT_TOTAL_ATTEMPTS
- set_blocked.assert_not_called()
+ extend_block.assert_not_called()
assert sleep.await_count == DEFAULT_UPSTREAM_MAX_RETRIES
@pytest.mark.parametrize("status_code", [500, 502, 503, 504])
@@ -620,7 +747,7 @@ class TestProviderRateLimiter:
openrouter = ProviderRateLimiter(rate_limit=20, rate_window=30)
assert nim is not openrouter
- nim.set_blocked(1.0)
+ nim.extend_reactive_block(1.0)
assert nim.is_blocked() is True
assert openrouter.is_blocked() is False
diff --git a/uv.lock b/uv.lock
index 1eb909f0..d6d22317 100644
--- a/uv.lock
+++ b/uv.lock
@@ -561,7 +561,7 @@ wheels = [
[[package]]
name = "free-claude-code"
-version = "3.5.0"
+version = "3.5.1"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },