Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Fixed rate limiting issues

Alishahryar1fab66edcd351
diff --git a/providers/nvidia_nim/client.py b/providers/nvidia_nim/client.py
index eedcbde1..9269b5b4 100644
--- a/providers/nvidia_nim/client.py
+++ b/providers/nvidia_nim/client.py
@@ -40,7 +40,7 @@ class NvidiaNimProvider(BaseProvider):
self._client = AsyncOpenAI(
api_key=self._api_key,
base_url=self._base_url,
- max_retries=2,
+ max_retries=0,
timeout=300.0,
)
@@ -52,27 +52,15 @@ class NvidiaNimProvider(BaseProvider):
self, request: Any, input_tokens: int = 0
) -> AsyncIterator[str]:
"""Stream response in Anthropic SSE format."""
- # Wait if globally rate limited
- waited_reactively = await self._global_rate_limiter.wait_if_blocked()
-
message_id = f"msg_{uuid.uuid4()}"
sse = SSEBuilder(message_id, request.model, input_tokens)
- if waited_reactively:
- error_msg = "⏱️ Global rate limit active. Resuming now..."
- logger.info("NIM_STREAM: Reactive block detected, notified user")
- yield sse.message_start()
- for event in sse.emit_error(error_msg):
- yield event
-
body = self._build_request_body(request, stream=True)
logger.info(
f"NIM_STREAM: model={body.get('model')} msgs={len(body.get('messages', []))} tools={len(body.get('tools', []))}"
)
- # Only emit message_start if we haven't already emitted it during reactive wait
- if not waited_reactively:
- yield sse.message_start()
+ yield sse.message_start()
think_parser = ThinkTagParser()
heuristic_parser = HeuristicToolParser()
@@ -83,7 +71,9 @@ class NvidiaNimProvider(BaseProvider):
error_message = ""
try:
- stream = await self._client.chat.completions.create(**body, stream=True)
+ stream = await self._global_rate_limiter.execute_with_retry(
+ self._client.chat.completions.create, **body, stream=True
+ )
async for chunk in stream:
# OpenAI client returns objects, not JSON
if getattr(chunk, "usage", None):
@@ -223,16 +213,16 @@ class NvidiaNimProvider(BaseProvider):
async def complete(self, request: Any) -> dict:
"""Make a non-streaming completion request."""
- await self._global_rate_limiter.wait_if_blocked()
-
body = self._build_request_body(request, stream=False)
logger.info(
f"NIM_COMPLETE: model={body.get('model')} msgs={len(body.get('messages', []))} tools={len(body.get('tools', []))}"
)
try:
- response = await self._client.chat.completions.create(**body)
- # Responseconverter expects a dict
+ response = await self._global_rate_limiter.execute_with_retry(
+ self._client.chat.completions.create, **body
+ )
+ # Response converter expects a dict
return response.model_dump()
except Exception as e:
logger.error(f"NIM_ERROR: {type(e).__name__}: {e}")
diff --git a/providers/rate_limit.py b/providers/rate_limit.py
index d2bfeec2..6b1b6104 100644
--- a/providers/rate_limit.py
+++ b/providers/rate_limit.py
@@ -1,11 +1,16 @@
"""Global rate limiter for API requests."""
import asyncio
+import random
import time
import logging
-from typing import Optional
+from typing import Any, Callable, Optional, TypeVar
+
+import openai
from aiolimiter import AsyncLimiter
+T = TypeVar("T")
+
logger = logging.getLogger(__name__)
@@ -98,3 +103,57 @@ class GlobalRateLimiter:
def remaining_wait(self) -> float:
"""Get remaining reactive wait time in seconds."""
return max(0, self._blocked_until - time.time())
+
+ async def execute_with_retry(
+ self,
+ fn: Callable[..., Any],
+ *args: Any,
+ max_retries: int = 3,
+ base_delay: float = 2.0,
+ max_delay: float = 60.0,
+ jitter: float = 1.0,
+ **kwargs: Any,
+ ) -> Any:
+ """Execute an async callable with rate limiting and retry on 429.
+
+ Waits for the token bucket before each attempt. On 429, applies
+ exponential backoff with jitter before retrying.
+
+ Args:
+ fn: Async callable to execute.
+ max_retries: Maximum number of retry attempts after the first failure.
+ base_delay: Base delay in seconds for exponential backoff.
+ max_delay: Maximum delay cap in seconds.
+ jitter: Maximum random jitter in seconds added to each delay.
+
+ Returns:
+ The result of the callable.
+
+ Raises:
+ The last exception if all retries are exhausted.
+ """
+ last_exc: Optional[Exception] = None
+
+ for attempt in range(1 + max_retries):
+ await self.wait_if_blocked()
+
+ try:
+ return await fn(*args, **kwargs)
+ except openai.RateLimitError as e:
+ last_exc = e
+ if attempt >= max_retries:
+ logger.warning(
+ f"Rate limit retry exhausted after {max_retries} retries"
+ )
+ break
+
+ delay = min(base_delay * (2 ** attempt), max_delay)
+ delay += random.uniform(0, jitter)
+ logger.warning(
+ f"Rate limited (429), attempt {attempt + 1}/{max_retries + 1}. "
+ f"Retrying in {delay:.1f}s..."
+ )
+ self.set_blocked(delay)
+ await asyncio.sleep(delay)
+
+ raise last_exc # type: ignore[misc]
diff --git a/tests/test_nvidia_nim.py b/tests/test_nvidia_nim.py
index b9bf73e5..3198c60f 100644
--- a/tests/test_nvidia_nim.py
+++ b/tests/test_nvidia_nim.py
@@ -42,6 +42,11 @@ def mock_rate_limiter():
with patch("providers.nvidia_nim.client.GlobalRateLimiter") as mock:
instance = mock.get_instance.return_value
instance.wait_if_blocked = AsyncMock(return_value=False)
+ # execute_with_retry should call through to the actual function
+ async def _passthrough(fn, *args, **kwargs):
+ return await fn(*args, **kwargs)
+
+ instance.execute_with_retry = AsyncMock(side_effect=_passthrough)
yield instance
diff --git a/tests/test_streaming_errors.py b/tests/test_streaming_errors.py
index fa330951..298d59fc 100644
--- a/tests/test_streaming_errors.py
+++ b/tests/test_streaming_errors.py
@@ -233,8 +233,8 @@ class TestStreamingExceptionHandling:
assert "The answer" in event_text
@pytest.mark.asyncio
- async def test_stream_rate_limited_shows_notice(self):
- """When globally rate limited, a notice is shown before stream starts."""
+ async def test_stream_rate_limited_retries_via_execute_with_retry(self):
+ """When rate limited, execute_with_retry handles retries transparently."""
provider = _make_provider()
request = _make_request()
@@ -248,16 +248,19 @@ class TestStreamingExceptionHandling:
new_callable=AsyncMock,
return_value=stream_mock,
):
+ # Mock execute_with_retry to pass through to the actual function
+ async def _passthrough(fn, *args, **kwargs):
+ return await fn(*args, **kwargs)
+
with patch.object(
provider._global_rate_limiter,
- "wait_if_blocked",
+ "execute_with_retry",
new_callable=AsyncMock,
- return_value=True,
+ side_effect=_passthrough,
):
events = await _collect_stream(provider, request)
event_text = "".join(events)
- assert "rate limit" in event_text.lower()
assert "Response" in event_text