Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Migrated from token bucket rate limiter to sliding window rate limiter

Alishahryar1665e24e2dbaf
diff --git a/providers/rate_limit.py b/providers/rate_limit.py
index 6b1b6104..88413e4a 100644
--- a/providers/rate_limit.py
+++ b/providers/rate_limit.py
@@ -1,13 +1,13 @@
"""Global rate limiter for API requests."""
import asyncio
+from collections import deque
import random
import time
import logging
from typing import Any, Callable, Optional, TypeVar
import openai
-from aiolimiter import AsyncLimiter
T = TypeVar("T")
@@ -18,7 +18,7 @@ class GlobalRateLimiter:
"""
Global singleton rate limiter that blocks all requests
when a rate limit error is encountered (reactive) and
- throttles requests (proactive) using aiolimiter.
+ throttles requests (proactive) using a strict rolling window.
Proactive limits - throttles requests to stay within API limits.
Reactive limits - pauses all requests when a 429 is hit.
@@ -31,7 +31,15 @@ class GlobalRateLimiter:
if hasattr(self, "_initialized"):
return
- self.limiter = AsyncLimiter(rate_limit, rate_window)
+ if rate_limit <= 0:
+ raise ValueError("rate_limit must be > 0")
+ if rate_window <= 0:
+ raise ValueError("rate_window must be > 0")
+
+ self._rate_limit = rate_limit
+ self._rate_window = float(rate_window)
+ # Monotonic timestamps of the last granted slots.
+ self._request_times: deque[float] = deque()
self._blocked_until: float = 0
self._lock = asyncio.Lock()
self._initialized = True
@@ -73,7 +81,7 @@ class GlobalRateLimiter:
"""
# 1. Reactive check: Wait if someone hit a 429
waited_reactively = False
- now = time.time()
+ now = time.monotonic()
if now < self._blocked_until:
wait_time = self._blocked_until - now
logger.warning(
@@ -82,9 +90,38 @@ class GlobalRateLimiter:
await asyncio.sleep(wait_time)
waited_reactively = True
- # 2. Proactive check: Acquire slot from aiolimiter
- async with self.limiter:
- return waited_reactively
+ # 2. Proactive check: strict rolling window (no bursts beyond N in last W seconds)
+ await self._acquire_proactive_slot()
+ return waited_reactively
+
+ async def _acquire_proactive_slot(self) -> None:
+ """
+ Acquire a proactive slot enforcing a strict rolling window.
+
+ Guarantees: at most `self._rate_limit` acquisitions in any interval of length
+ `self._rate_window` (seconds).
+ """
+ while True:
+ wait_time = 0.0
+ async with self._lock:
+ now = time.monotonic()
+ cutoff = now - self._rate_window
+
+ while self._request_times and self._request_times[0] <= cutoff:
+ self._request_times.popleft()
+
+ if len(self._request_times) < self._rate_limit:
+ self._request_times.append(now)
+ return
+
+ oldest = self._request_times[0]
+ wait_time = max(0.0, (oldest + self._rate_window) - now)
+
+ # Sleep outside the lock so other tasks can continue to queue.
+ if wait_time > 0:
+ await asyncio.sleep(wait_time)
+ else:
+ await asyncio.sleep(0)
def set_blocked(self, seconds: float = 60) -> None:
"""
@@ -93,16 +130,16 @@ class GlobalRateLimiter:
Args:
seconds: How long to block (default 60s)
"""
- self._blocked_until = time.time() + seconds
+ self._blocked_until = time.monotonic() + seconds
logger.warning(f"Global provider rate limit set for {seconds:.1f}s (reactive)")
def is_blocked(self) -> bool:
"""Check if currently reactively blocked."""
- return time.time() < self._blocked_until
+ return time.monotonic() < self._blocked_until
def remaining_wait(self) -> float:
"""Get remaining reactive wait time in seconds."""
- return max(0, self._blocked_until - time.time())
+ return max(0.0, self._blocked_until - time.monotonic())
async def execute_with_retry(
self,
@@ -116,7 +153,7 @@ class GlobalRateLimiter:
) -> Any:
"""Execute an async callable with rate limiting and retry on 429.
- Waits for the token bucket before each attempt. On 429, applies
+ Waits for the proactive limiter before each attempt. On 429, applies
exponential backoff with jitter before retrying.
Args:
@@ -147,7 +184,7 @@ class GlobalRateLimiter:
)
break
- delay = min(base_delay * (2 ** attempt), max_delay)
+ delay = min(base_delay * (2**attempt), max_delay)
delay += random.uniform(0, jitter)
logger.warning(
f"Rate limited (429), attempt {attempt + 1}/{max_retries + 1}. "
diff --git a/tests/test_nvidia_nim.py b/tests/test_nvidia_nim.py
index 3198c60f..65d77d44 100644
--- a/tests/test_nvidia_nim.py
+++ b/tests/test_nvidia_nim.py
@@ -42,6 +42,7 @@ def mock_rate_limiter():
with patch("providers.nvidia_nim.client.GlobalRateLimiter") as mock:
instance = mock.get_instance.return_value
instance.wait_if_blocked = AsyncMock(return_value=False)
+
# execute_with_retry should call through to the actual function
async def _passthrough(fn, *args, **kwargs):
return await fn(*args, **kwargs)
diff --git a/tests/test_provider_rate_limit.py b/tests/test_provider_rate_limit.py
index 143e300e..9c403a46 100644
--- a/tests/test_provider_rate_limit.py
+++ b/tests/test_provider_rate_limit.py
@@ -25,7 +25,7 @@ class TestProviderRateLimiter:
@pytest.mark.asyncio
async def test_proactive_throttling(self):
"""
- Test proactive throttling using aiolimiter.
+ Test proactive throttling.
Logic ported from verify_provider_limiter.py
"""
# Re-init with tight limits: 1 request per 0.25 second
@@ -157,3 +157,35 @@ class TestProviderRateLimiter:
limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)
result = await limiter.wait_if_blocked()
assert result is False
+
+ @pytest.mark.asyncio
+ async def test_proactive_strict_rolling_window(self):
+ """
+ Proactive limiter should enforce a strict rolling window:
+ for any i, t[i+rate_limit] - t[i] >= rate_window (within tolerance).
+ """
+ GlobalRateLimiter.reset_instance()
+ rate_limit = 2
+ rate_window = 0.5
+ limiter = GlobalRateLimiter.get_instance(
+ rate_limit=rate_limit, rate_window=rate_window
+ )
+
+ acquired: list[float] = []
+
+ async def acquire():
+ await limiter.wait_if_blocked()
+ acquired.append(time.monotonic())
+
+ # Trigger concurrency; without strict rolling windows, this can burst.
+ await asyncio.gather(*(acquire() for _ in range(5)))
+
+ acquired.sort()
+ assert len(acquired) == 5
+
+ tolerance = 0.05
+ for i in range(len(acquired) - rate_limit):
+ assert acquired[i + rate_limit] - acquired[i] >= rate_window - tolerance, (
+ f"Rolling window violated at i={i}: "
+ f"dt={acquired[i + rate_limit] - acquired[i]:.3f}s"
+ )