Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Fixed rate limiting issues
Alishahryar1fab66edcd351
diff --git a/providers/nvidia_nim/client.py b/providers/nvidia_nim/client.pyindex eedcbde1..9269b5b4 100644--- a/providers/nvidia_nim/client.py+++ b/providers/nvidia_nim/client.py@@ -40,7 +40,7 @@ class NvidiaNimProvider(BaseProvider):self._client = AsyncOpenAI(api_key=self._api_key,base_url=self._base_url,- max_retries=2,+ max_retries=0,timeout=300.0,)@@ -52,27 +52,15 @@ class NvidiaNimProvider(BaseProvider):self, request: Any, input_tokens: int = 0) -> AsyncIterator[str]:"""Stream response in Anthropic SSE format."""- # Wait if globally rate limited- waited_reactively = await self._global_rate_limiter.wait_if_blocked()-message_id = f"msg_{uuid.uuid4()}"sse = SSEBuilder(message_id, request.model, input_tokens)- if waited_reactively:- error_msg = "⏱️ Global rate limit active. Resuming now..."- logger.info("NIM_STREAM: Reactive block detected, notified user")- yield sse.message_start()- for event in sse.emit_error(error_msg):- yield event-body = self._build_request_body(request, stream=True)logger.info(f"NIM_STREAM: model={body.get('model')} msgs={len(body.get('messages', []))} tools={len(body.get('tools', []))}")- # Only emit message_start if we haven't already emitted it during reactive wait- if not waited_reactively:- yield sse.message_start()+ yield sse.message_start()think_parser = ThinkTagParser()heuristic_parser = HeuristicToolParser()@@ -83,7 +71,9 @@ class NvidiaNimProvider(BaseProvider):error_message = ""try:- stream = await self._client.chat.completions.create(**body, stream=True)+ stream = await self._global_rate_limiter.execute_with_retry(+ self._client.chat.completions.create, **body, stream=True+ )async for chunk in stream:# OpenAI client returns objects, not JSONif getattr(chunk, "usage", None):@@ -223,16 +213,16 @@ class NvidiaNimProvider(BaseProvider):async def complete(self, request: Any) -> dict:"""Make a non-streaming completion request."""- await self._global_rate_limiter.wait_if_blocked()-body = self._build_request_body(request, stream=False)logger.info(f"NIM_COMPLETE: model={body.get('model')} msgs={len(body.get('messages', []))} tools={len(body.get('tools', []))}")try:- response = await self._client.chat.completions.create(**body)- # Responseconverter expects a dict+ response = await self._global_rate_limiter.execute_with_retry(+ self._client.chat.completions.create, **body+ )+ # Response converter expects a dictreturn response.model_dump()except Exception as e:logger.error(f"NIM_ERROR: {type(e).__name__}: {e}")diff --git a/providers/rate_limit.py b/providers/rate_limit.pyindex d2bfeec2..6b1b6104 100644--- a/providers/rate_limit.py+++ b/providers/rate_limit.py@@ -1,11 +1,16 @@"""Global rate limiter for API requests."""import asyncio+import randomimport timeimport logging-from typing import Optional+from typing import Any, Callable, Optional, TypeVar++import openaifrom aiolimiter import AsyncLimiter+T = TypeVar("T")+logger = logging.getLogger(__name__)@@ -98,3 +103,57 @@ class GlobalRateLimiter:def remaining_wait(self) -> float:"""Get remaining reactive wait time in seconds."""return max(0, self._blocked_until - time.time())++ async def execute_with_retry(+ self,+ fn: Callable[..., Any],+ *args: Any,+ max_retries: int = 3,+ base_delay: float = 2.0,+ max_delay: float = 60.0,+ jitter: float = 1.0,+ **kwargs: Any,+ ) -> Any:+ """Execute an async callable with rate limiting and retry on 429.++ Waits for the token bucket before each attempt. On 429, applies+ exponential backoff with jitter before retrying.++ Args:+ fn: Async callable to execute.+ max_retries: Maximum number of retry attempts after the first failure.+ base_delay: Base delay in seconds for exponential backoff.+ max_delay: Maximum delay cap in seconds.+ jitter: Maximum random jitter in seconds added to each delay.++ Returns:+ The result of the callable.++ Raises:+ The last exception if all retries are exhausted.+ """+ last_exc: Optional[Exception] = None++ for attempt in range(1 + max_retries):+ await self.wait_if_blocked()++ try:+ return await fn(*args, **kwargs)+ except openai.RateLimitError as e:+ last_exc = e+ if attempt >= max_retries:+ logger.warning(+ f"Rate limit retry exhausted after {max_retries} retries"+ )+ break++ delay = min(base_delay * (2 ** attempt), max_delay)+ delay += random.uniform(0, jitter)+ logger.warning(+ f"Rate limited (429), attempt {attempt + 1}/{max_retries + 1}. "+ f"Retrying in {delay:.1f}s..."+ )+ self.set_blocked(delay)+ await asyncio.sleep(delay)++ raise last_exc # type: ignore[misc]diff --git a/tests/test_nvidia_nim.py b/tests/test_nvidia_nim.pyindex b9bf73e5..3198c60f 100644--- a/tests/test_nvidia_nim.py+++ b/tests/test_nvidia_nim.py@@ -42,6 +42,11 @@ def mock_rate_limiter():with patch("providers.nvidia_nim.client.GlobalRateLimiter") as mock:instance = mock.get_instance.return_valueinstance.wait_if_blocked = AsyncMock(return_value=False)+ # execute_with_retry should call through to the actual function+ async def _passthrough(fn, *args, **kwargs):+ return await fn(*args, **kwargs)++ instance.execute_with_retry = AsyncMock(side_effect=_passthrough)yield instancediff --git a/tests/test_streaming_errors.py b/tests/test_streaming_errors.pyindex fa330951..298d59fc 100644--- a/tests/test_streaming_errors.py+++ b/tests/test_streaming_errors.py@@ -233,8 +233,8 @@ class TestStreamingExceptionHandling:assert "The answer" in event_text@pytest.mark.asyncio- async def test_stream_rate_limited_shows_notice(self):- """When globally rate limited, a notice is shown before stream starts."""+ async def test_stream_rate_limited_retries_via_execute_with_retry(self):+ """When rate limited, execute_with_retry handles retries transparently."""provider = _make_provider()request = _make_request()@@ -248,16 +248,19 @@ class TestStreamingExceptionHandling:new_callable=AsyncMock,return_value=stream_mock,):+ # Mock execute_with_retry to pass through to the actual function+ async def _passthrough(fn, *args, **kwargs):+ return await fn(*args, **kwargs)+with patch.object(provider._global_rate_limiter,- "wait_if_blocked",+ "execute_with_retry",new_callable=AsyncMock,- return_value=True,+ side_effect=_passthrough,):events = await _collect_stream(provider, request)event_text = "".join(events)- assert "rate limit" in event_text.lower()assert "Response" in event_text