Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Migrated from token bucket rate limiter to sliding window rate limiter
Alishahryar1665e24e2dbaf
diff --git a/providers/rate_limit.py b/providers/rate_limit.pyindex 6b1b6104..88413e4a 100644--- a/providers/rate_limit.py+++ b/providers/rate_limit.py@@ -1,13 +1,13 @@"""Global rate limiter for API requests."""import asyncio+from collections import dequeimport randomimport timeimport loggingfrom typing import Any, Callable, Optional, TypeVarimport openai-from aiolimiter import AsyncLimiterT = TypeVar("T")@@ -18,7 +18,7 @@ class GlobalRateLimiter:"""Global singleton rate limiter that blocks all requestswhen a rate limit error is encountered (reactive) and- throttles requests (proactive) using aiolimiter.+ throttles requests (proactive) using a strict rolling window.Proactive limits - throttles requests to stay within API limits.Reactive limits - pauses all requests when a 429 is hit.@@ -31,7 +31,15 @@ class GlobalRateLimiter:if hasattr(self, "_initialized"):return- self.limiter = AsyncLimiter(rate_limit, rate_window)+ if rate_limit <= 0:+ raise ValueError("rate_limit must be > 0")+ if rate_window <= 0:+ raise ValueError("rate_window must be > 0")++ self._rate_limit = rate_limit+ self._rate_window = float(rate_window)+ # Monotonic timestamps of the last granted slots.+ self._request_times: deque[float] = deque()self._blocked_until: float = 0self._lock = asyncio.Lock()self._initialized = True@@ -73,7 +81,7 @@ class GlobalRateLimiter:"""# 1. Reactive check: Wait if someone hit a 429waited_reactively = False- now = time.time()+ now = time.monotonic()if now < self._blocked_until:wait_time = self._blocked_until - nowlogger.warning(@@ -82,9 +90,38 @@ class GlobalRateLimiter:await asyncio.sleep(wait_time)waited_reactively = True- # 2. Proactive check: Acquire slot from aiolimiter- async with self.limiter:- return waited_reactively+ # 2. Proactive check: strict rolling window (no bursts beyond N in last W seconds)+ await self._acquire_proactive_slot()+ return waited_reactively++ async def _acquire_proactive_slot(self) -> None:+ """+ Acquire a proactive slot enforcing a strict rolling window.++ Guarantees: at most `self._rate_limit` acquisitions in any interval of length+ `self._rate_window` (seconds).+ """+ while True:+ wait_time = 0.0+ async with self._lock:+ now = time.monotonic()+ cutoff = now - self._rate_window++ while self._request_times and self._request_times[0] <= cutoff:+ self._request_times.popleft()++ if len(self._request_times) < self._rate_limit:+ self._request_times.append(now)+ return++ oldest = self._request_times[0]+ wait_time = max(0.0, (oldest + self._rate_window) - now)++ # Sleep outside the lock so other tasks can continue to queue.+ if wait_time > 0:+ await asyncio.sleep(wait_time)+ else:+ await asyncio.sleep(0)def set_blocked(self, seconds: float = 60) -> None:"""@@ -93,16 +130,16 @@ class GlobalRateLimiter:Args:seconds: How long to block (default 60s)"""- self._blocked_until = time.time() + seconds+ self._blocked_until = time.monotonic() + secondslogger.warning(f"Global provider rate limit set for {seconds:.1f}s (reactive)")def is_blocked(self) -> bool:"""Check if currently reactively blocked."""- return time.time() < self._blocked_until+ return time.monotonic() < self._blocked_untildef remaining_wait(self) -> float:"""Get remaining reactive wait time in seconds."""- return max(0, self._blocked_until - time.time())+ return max(0.0, self._blocked_until - time.monotonic())async def execute_with_retry(self,@@ -116,7 +153,7 @@ class GlobalRateLimiter:) -> Any:"""Execute an async callable with rate limiting and retry on 429.- Waits for the token bucket before each attempt. On 429, applies+ Waits for the proactive limiter before each attempt. On 429, appliesexponential backoff with jitter before retrying.Args:@@ -147,7 +184,7 @@ class GlobalRateLimiter:)break- delay = min(base_delay * (2 ** attempt), max_delay)+ delay = min(base_delay * (2**attempt), max_delay)delay += random.uniform(0, jitter)logger.warning(f"Rate limited (429), attempt {attempt + 1}/{max_retries + 1}. "diff --git a/tests/test_nvidia_nim.py b/tests/test_nvidia_nim.pyindex 3198c60f..65d77d44 100644--- a/tests/test_nvidia_nim.py+++ b/tests/test_nvidia_nim.py@@ -42,6 +42,7 @@ def mock_rate_limiter():with patch("providers.nvidia_nim.client.GlobalRateLimiter") as mock:instance = mock.get_instance.return_valueinstance.wait_if_blocked = AsyncMock(return_value=False)+# execute_with_retry should call through to the actual functionasync def _passthrough(fn, *args, **kwargs):return await fn(*args, **kwargs)diff --git a/tests/test_provider_rate_limit.py b/tests/test_provider_rate_limit.pyindex 143e300e..9c403a46 100644--- a/tests/test_provider_rate_limit.py+++ b/tests/test_provider_rate_limit.py@@ -25,7 +25,7 @@ class TestProviderRateLimiter:@pytest.mark.asyncioasync def test_proactive_throttling(self):"""- Test proactive throttling using aiolimiter.+ Test proactive throttling.Logic ported from verify_provider_limiter.py"""# Re-init with tight limits: 1 request per 0.25 second@@ -157,3 +157,35 @@ class TestProviderRateLimiter:limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)result = await limiter.wait_if_blocked()assert result is False++ @pytest.mark.asyncio+ async def test_proactive_strict_rolling_window(self):+ """+ Proactive limiter should enforce a strict rolling window:+ for any i, t[i+rate_limit] - t[i] >= rate_window (within tolerance).+ """+ GlobalRateLimiter.reset_instance()+ rate_limit = 2+ rate_window = 0.5+ limiter = GlobalRateLimiter.get_instance(+ rate_limit=rate_limit, rate_window=rate_window+ )++ acquired: list[float] = []++ async def acquire():+ await limiter.wait_if_blocked()+ acquired.append(time.monotonic())++ # Trigger concurrency; without strict rolling windows, this can burst.+ await asyncio.gather(*(acquire() for _ in range(5)))++ acquired.sort()+ assert len(acquired) == 5++ tolerance = 0.05+ for i in range(len(acquired) - rate_limit):+ assert acquired[i + rate_limit] - acquired[i] >= rate_window - tolerance, (+ f"Rolling window violated at i={i}: "+ f"dt={acquired[i + rate_limit] - acquired[i]:.3f}s"+ )