Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(providers): generous rate-limit defaults + retry-after-aware 429 backoff (#417)

Raghav Chamadiyab08656f0b685
diff --git a/packages/core/src/repowise/core/providers/llm/anthropic.py b/packages/core/src/repowise/core/providers/llm/anthropic.py
index f2f64c2b5..46f4e2292 100644
--- a/packages/core/src/repowise/core/providers/llm/anthropic.py
+++ b/packages/core/src/repowise/core/providers/llm/anthropic.py
@@ -21,13 +21,7 @@ import structlog
from anthropic import APIStatusError as _AnthropicAPIStatusError
from anthropic import AsyncAnthropic
from anthropic import RateLimitError as _AnthropicRateLimitError
-from tenacity import (
- RetryError,
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential_jitter,
-)
+from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
@@ -40,6 +34,10 @@ from repowise.core.providers.llm.base import (
RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
+ parse_retry_after,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode
@@ -49,9 +47,6 @@ if TYPE_CHECKING:
log = structlog.get_logger(__name__)
-_MAX_RETRIES = 3
-_MIN_WAIT = 1.0
-_MAX_WAIT = 4.0
_DEFAULT_BASE_URL = "https://api.anthropic.com"
@@ -182,13 +177,13 @@ class AnthropicProvider(BaseProvider):
except RetryError as exc:
raise ProviderError(
"anthropic",
- f"All {_MAX_RETRIES} retries exhausted: {exc}",
+ f"All retries exhausted: {exc}",
) from exc
@retry(
- retry=retry_if_exception_type(ProviderError),
- stop=stop_after_attempt(_MAX_RETRIES),
- wait=wait_exponential_jitter(initial=_MIN_WAIT, max=_MAX_WAIT),
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
reraise=True,
)
async def _generate_with_retry(
@@ -212,7 +207,14 @@ class AnthropicProvider(BaseProvider):
messages=messages_param,
)
except _AnthropicRateLimitError as exc:
- raise RateLimitError("anthropic", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "anthropic",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _AnthropicAPIStatusError as exc:
raise ProviderError("anthropic", str(exc), status_code=exc.status_code) from exc
@@ -352,7 +354,14 @@ class AnthropicProvider(BaseProvider):
elif event.type == "message_stop":
pass # Final cleanup; stop already yielded via message_delta
except _AnthropicRateLimitError as exc:
- raise RateLimitError("anthropic", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "anthropic",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _AnthropicAPIStatusError as exc:
raise ProviderError("anthropic", str(exc), status_code=exc.status_code) from exc
diff --git a/packages/core/src/repowise/core/providers/llm/base.py b/packages/core/src/repowise/core/providers/llm/base.py
index 96dc3146c..74fece5db 100644
--- a/packages/core/src/repowise/core/providers/llm/base.py
+++ b/packages/core/src/repowise/core/providers/llm/base.py
@@ -232,12 +232,115 @@ class ProviderError(Exception):
class RateLimitError(ProviderError):
- """Raised when rate limits are permanently exhausted after retries.
+ """Raised on a 429 response from the provider.
- This is a sub-class of ProviderError. Callers can catch either,
- but RateLimitError signals that backing off longer won't help —
- the operator needs to review rate limits or reduce concurrency.
+ This is a sub-class of ProviderError. Callers can catch either.
+ Rate-limit 429s are *recoverable* — the shared retry policy
+ (``provider_retry_*`` below) backs off patiently (respecting the
+ provider's ``retry-after`` when known) before giving up.
+
+ Attributes:
+ retry_after: Seconds the provider asked us to wait before retrying,
+ parsed from the ``retry-after`` header (or provider
+ equivalent). ``None`` when the provider didn't say.
+ """
+
+ def __init__(
+ self,
+ provider: str,
+ message: str,
+ status_code: int | None = 429,
+ retry_after: float | None = None,
+ ) -> None:
+ self.retry_after = retry_after
+ super().__init__(provider, message, status_code=status_code)
+
+
+def parse_retry_after(headers: Any) -> float | None:
+ """Best-effort parse of a ``retry-after`` header value in seconds.
+
+ Accepts any mapping-like object with ``.get`` (httpx.Headers, dict).
+ Returns ``None`` when absent or unparseable. HTTP-date form is ignored —
+ providers we call send delta-seconds.
+ """
+ if headers is None:
+ return None
+ try:
+ raw = headers.get("retry-after") or headers.get("Retry-After")
+ if raw is None:
+ return None
+ seconds = float(raw)
+ return seconds if 0 < seconds <= 600 else None
+ except Exception:
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Shared tenacity retry policy for all HTTP providers
+#
+# Philosophy: the client-side RateLimiter is only a flood guard with very
+# generous defaults; the provider's own 429s are the real throttle signal.
+# 429s therefore get a patient, retry-after-aware backoff (they resolve
+# within the provider's rate window), while transient 5xx/network errors get
+# a short exponential retry, and non-retryable 4xx (400/401/403/404/422)
+# fail immediately instead of burning three pointless attempts.
+# ---------------------------------------------------------------------------
+
+_TRANSIENT_ATTEMPTS = 3 # 5xx / network / unknown errors
+_RATE_LIMIT_ATTEMPTS = 6 # 429s — waits 2+4+8+16+30 ≈ one full rate window
+_RATE_LIMIT_MAX_WAIT = 30.0 # per-attempt cap; cumulative covers the window
+_NON_RETRYABLE_STATUS = frozenset({400, 401, 403, 404, 405, 413, 422})
+
+# Multiplier applied to every computed wait. Production leaves this at 1.0;
+# tests shrink it (e.g. 0.01) so persistent-429 retry paths stay fast without
+# changing the retry shape. Read at call time, so monkeypatching works.
+_WAIT_SCALE = 1.0
+
+
+def _retry_exc(retry_state: Any) -> BaseException | None:
+ outcome = retry_state.outcome
+ return outcome.exception() if outcome is not None else None
+
+
+def provider_should_retry(retry_state: Any) -> bool:
+ """tenacity ``retry`` predicate: retry 429s and transient errors only."""
+ exc = _retry_exc(retry_state)
+ if not isinstance(exc, ProviderError):
+ return False
+ if isinstance(exc, RateLimitError):
+ return True
+ return exc.status_code not in _NON_RETRYABLE_STATUS
+
+
+def provider_retry_stop(retry_state: Any) -> bool:
+ """tenacity ``stop`` predicate: more attempts for 429s than for 5xx."""
+ limit = (
+ _RATE_LIMIT_ATTEMPTS
+ if isinstance(_retry_exc(retry_state), RateLimitError)
+ else _TRANSIENT_ATTEMPTS
+ )
+ return retry_state.attempt_number >= limit
+
+
+def provider_retry_wait(retry_state: Any) -> float:
+ """tenacity ``wait`` callable.
+
+ 429 with retry-after → honour it (+ jitter). 429 without → exponential
+ jitter, per-attempt cap 30s, cumulatively covering a full per-minute rate
+ window. Everything else → short exponential jitter (1-4s), matching the
+ old policy.
"""
+ import random
+
+ exc = _retry_exc(retry_state)
+ attempt = retry_state.attempt_number
+ if isinstance(exc, RateLimitError):
+ if exc.retry_after is not None:
+ wait = min(exc.retry_after + random.uniform(0, 1), 65.0)
+ else:
+ wait = min(2.0**attempt + random.uniform(0, 1), _RATE_LIMIT_MAX_WAIT)
+ return wait * _WAIT_SCALE
+ return (min(2.0 ** (attempt - 1), 4.0) * random.uniform(0.5, 1.0) + 0.5) * _WAIT_SCALE
def ensure_reasoning_supported(
diff --git a/packages/core/src/repowise/core/providers/llm/deepseek.py b/packages/core/src/repowise/core/providers/llm/deepseek.py
index fa12db48f..2e8fa0949 100644
--- a/packages/core/src/repowise/core/providers/llm/deepseek.py
+++ b/packages/core/src/repowise/core/providers/llm/deepseek.py
@@ -21,13 +21,7 @@ import structlog
from openai import APIStatusError as _OpenAIAPIStatusError
from openai import AsyncOpenAI
from openai import RateLimitError as _OpenAIRateLimitError
-from tenacity import (
- RetryError,
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential_jitter,
-)
+from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
@@ -39,6 +33,10 @@ from repowise.core.providers.llm.base import (
RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
+ parse_retry_after,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -48,10 +46,6 @@ if TYPE_CHECKING:
log = structlog.get_logger(__name__)
-_MAX_RETRIES = 3
-_MIN_WAIT = 1.0
-_MAX_WAIT = 4.0
-
_DEFAULT_BASE_URL = "https://api.deepseek.com"
_DEEPSEEK_REASONING_MODES: tuple[ReasoningMode, ...] = (
"off",
@@ -232,13 +226,13 @@ class DeepSeekProvider(BaseProvider):
except RetryError as exc:
raise ProviderError(
"deepseek",
- f"All {_MAX_RETRIES} retries exhausted: {exc}",
+ f"All retries exhausted: {exc}",
) from exc
@retry(
- retry=retry_if_exception_type(ProviderError),
- stop=stop_after_attempt(_MAX_RETRIES),
- wait=wait_exponential_jitter(initial=_MIN_WAIT, max=_MAX_WAIT),
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
reraise=True,
)
async def _generate_with_retry(
@@ -263,7 +257,14 @@ class DeepSeekProvider(BaseProvider):
kwargs.update(_deepseek_reasoning_kwargs(reasoning))
response = await self._client.chat.completions.create(**kwargs)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("deepseek", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "deepseek",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("deepseek", str(exc), status_code=exc.status_code) from exc
@@ -330,7 +331,14 @@ class DeepSeekProvider(BaseProvider):
try:
stream = await self._client.chat.completions.create(**kwargs)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("deepseek", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "deepseek",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("deepseek", str(exc), status_code=exc.status_code) from exc
@@ -392,6 +400,13 @@ class DeepSeekProvider(BaseProvider):
stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("deepseek", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "deepseek",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("deepseek", str(exc), status_code=exc.status_code) from exc
diff --git a/packages/core/src/repowise/core/providers/llm/gemini.py b/packages/core/src/repowise/core/providers/llm/gemini.py
index c8f03ae27..5255b73c0 100644
--- a/packages/core/src/repowise/core/providers/llm/gemini.py
+++ b/packages/core/src/repowise/core/providers/llm/gemini.py
@@ -18,13 +18,7 @@ from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
import structlog
-from tenacity import (
- RetryError,
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential_jitter,
-)
+from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
@@ -36,6 +30,9 @@ from repowise.core.providers.llm.base import (
RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -49,9 +46,6 @@ logging.getLogger("google_genai._api_client").setLevel(logging.ERROR)
log = structlog.get_logger(__name__)
-_MAX_RETRIES = 3
-_MIN_WAIT = 1.0
-_MAX_WAIT = 4.0
_DEFAULT_MODELS_URL = "https://generativelanguage.googleapis.com/v1beta/models"
_GEMINI_THINKING_MODES: tuple[ReasoningMode, ...] = (
"minimal",
@@ -62,6 +56,24 @@ _GEMINI_THINKING_MODES: tuple[ReasoningMode, ...] = (
_GEMINI_THINKING_MODELS_BY_BASE: dict[str, set[str]] = {}
+def _gemini_retry_delay(exc_str: str) -> float | None:
+ """Extract the ``retryDelay`` hint from a google-genai 429 error string.
+
+ Quota errors embed RetryInfo like ``'retryDelay': '21s'`` in the error
+ details. Best-effort: returns ``None`` when absent.
+ """
+ import re
+
+ match = re.search(r"retryDelay['\"]?\s*:\s*['\"]?(\d+(?:\.\d+)?)s", exc_str)
+ if match:
+ try:
+ seconds = float(match.group(1))
+ return seconds if 0 < seconds <= 600 else None
+ except ValueError:
+ return None
+ return None
+
+
def _gemini_cache_key(base_url: str | None) -> str:
return (base_url or _DEFAULT_MODELS_URL).rstrip("/")
@@ -281,13 +293,13 @@ class GeminiProvider(BaseProvider):
except RetryError as exc:
raise ProviderError(
"gemini",
- f"All {_MAX_RETRIES} retries exhausted: {exc}",
+ f"All retries exhausted: {exc}",
) from exc
@retry(
- retry=retry_if_exception_type(ProviderError),
- stop=stop_after_attempt(_MAX_RETRIES),
- wait=wait_exponential_jitter(initial=_MIN_WAIT, max=_MAX_WAIT),
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
reraise=True,
)
async def _generate_with_retry(
@@ -354,7 +366,12 @@ class GeminiProvider(BaseProvider):
exc_str = str(exc)
status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
if status_code == 429 or "429" in exc_str or "quota" in exc_str.lower():
- raise RateLimitError("gemini", exc_str, status_code=429) from exc
+ raise RateLimitError(
+ "gemini",
+ exc_str,
+ status_code=429,
+ retry_after=_gemini_retry_delay(exc_str),
+ ) from exc
raise ProviderError("gemini", f"{type(exc).__name__}: {exc_str}") from exc
usage = response.usage_metadata
@@ -450,7 +467,12 @@ class GeminiProvider(BaseProvider):
exc_str = str(exc)
status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
if status_code == 429 or "429" in exc_str or "quota" in exc_str.lower():
- raise RateLimitError("gemini", exc_str, status_code=429) from exc
+ raise RateLimitError(
+ "gemini",
+ exc_str,
+ status_code=429,
+ retry_after=_gemini_retry_delay(exc_str),
+ ) from exc
raise ProviderError("gemini", f"{type(exc).__name__}: {exc_str}") from exc
return response
diff --git a/packages/core/src/repowise/core/providers/llm/litellm.py b/packages/core/src/repowise/core/providers/llm/litellm.py
index 064f29938..2a25a1f86 100644
--- a/packages/core/src/repowise/core/providers/llm/litellm.py
+++ b/packages/core/src/repowise/core/providers/llm/litellm.py
@@ -25,13 +25,7 @@ from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
import structlog
-from tenacity import (
- RetryError,
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential_jitter,
-)
+from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
@@ -43,6 +37,10 @@ from repowise.core.providers.llm.base import (
RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
+ parse_retry_after,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -52,9 +50,6 @@ if TYPE_CHECKING:
log = structlog.get_logger(__name__)
-_MAX_RETRIES = 3
-_MIN_WAIT = 1.0
-_MAX_WAIT = 4.0
_LITELLM_REASONING_MODES: tuple[ReasoningMode, ...] = ("low", "medium", "high")
@@ -210,13 +205,13 @@ class LiteLLMProvider(BaseProvider):
except RetryError as exc:
raise ProviderError(
"litellm",
- f"All {_MAX_RETRIES} retries exhausted: {exc}",
+ f"All retries exhausted: {exc}",
) from exc
@retry(
- retry=retry_if_exception_type(ProviderError),
- stop=stop_after_attempt(_MAX_RETRIES),
- wait=wait_exponential_jitter(initial=_MIN_WAIT, max=_MAX_WAIT),
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
reraise=True,
)
async def _generate_with_retry(
@@ -253,7 +248,14 @@ class LiteLLMProvider(BaseProvider):
try:
response = await litellm.acompletion(**call_kwargs)
except litellm.RateLimitError as exc:
- raise RateLimitError("litellm", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "litellm",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except litellm.APIError as exc:
raise ProviderError("litellm", str(exc)) from exc
except Exception as exc:
@@ -330,7 +332,14 @@ class LiteLLMProvider(BaseProvider):
try:
stream = await litellm.acompletion(**call_kwargs)
except litellm.RateLimitError as exc:
- raise RateLimitError("litellm", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "litellm",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except litellm.APIError as exc:
raise ProviderError("litellm", str(exc)) from exc
@@ -382,6 +391,13 @@ class LiteLLMProvider(BaseProvider):
stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
except litellm.RateLimitError as exc:
- raise RateLimitError("litellm", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "litellm",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except Exception as exc:
raise ProviderError("litellm", f"{type(exc).__name__}: {exc}") from exc
diff --git a/packages/core/src/repowise/core/providers/llm/openai.py b/packages/core/src/repowise/core/providers/llm/openai.py
index 0ed5eea13..483118ef1 100644
--- a/packages/core/src/repowise/core/providers/llm/openai.py
+++ b/packages/core/src/repowise/core/providers/llm/openai.py
@@ -21,13 +21,7 @@ import structlog
from openai import APIStatusError as _OpenAIAPIStatusError
from openai import AsyncOpenAI
from openai import RateLimitError as _OpenAIRateLimitError
-from tenacity import (
- RetryError,
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential_jitter,
-)
+from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
@@ -40,6 +34,10 @@ from repowise.core.providers.llm.base import (
RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
+ parse_retry_after,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -49,9 +47,6 @@ if TYPE_CHECKING:
log = structlog.get_logger(__name__)
-_MAX_RETRIES = 3
-_MIN_WAIT = 1.0
-_MAX_WAIT = 4.0
_QWEN_THINKING_MODEL_MARKERS = ("qwen", "qwq")
_OPENAI_TEXT_MODEL_PREFIXES = ("gpt-", "o1", "o3", "o4")
_OPENAI_NON_TEXT_MARKERS = (
@@ -286,13 +281,13 @@ class OpenAIProvider(BaseProvider):
except RetryError as exc:
raise ProviderError(
"openai",
- f"All {_MAX_RETRIES} retries exhausted: {exc}",
+ f"All retries exhausted: {exc}",
) from exc
@retry(
- retry=retry_if_exception_type(ProviderError),
- stop=stop_after_attempt(_MAX_RETRIES),
- wait=wait_exponential_jitter(initial=_MIN_WAIT, max=_MAX_WAIT),
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
reraise=True,
)
async def _generate_with_retry(
@@ -319,7 +314,14 @@ class OpenAIProvider(BaseProvider):
**kwargs,
)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("openai", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "openai",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("openai", str(exc), status_code=exc.status_code) from exc
@@ -394,7 +396,14 @@ class OpenAIProvider(BaseProvider):
try:
stream = await self._client.chat.completions.create(**kwargs)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("openai", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "openai",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("openai", str(exc), status_code=exc.status_code) from exc
@@ -460,6 +469,13 @@ class OpenAIProvider(BaseProvider):
stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("openai", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "openai",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("openai", str(exc), status_code=exc.status_code) from exc
diff --git a/packages/core/src/repowise/core/providers/llm/openrouter.py b/packages/core/src/repowise/core/providers/llm/openrouter.py
index 0c25c7b08..689fd0488 100644
--- a/packages/core/src/repowise/core/providers/llm/openrouter.py
+++ b/packages/core/src/repowise/core/providers/llm/openrouter.py
@@ -21,13 +21,7 @@ import structlog
from openai import APIStatusError as _OpenAIAPIStatusError
from openai import AsyncOpenAI
from openai import RateLimitError as _OpenAIRateLimitError
-from tenacity import (
- RetryError,
- retry,
- retry_if_exception_type,
- stop_after_attempt,
- wait_exponential_jitter,
-)
+from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
@@ -39,6 +33,10 @@ from repowise.core.providers.llm.base import (
RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
+ parse_retry_after,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -48,9 +46,6 @@ if TYPE_CHECKING:
log = structlog.get_logger(__name__)
-_MAX_RETRIES = 3
-_MIN_WAIT = 1.0
-_MAX_WAIT = 4.0
_OPENROUTER_REASONING_MODES: tuple[ReasoningMode, ...] = (
"off",
"none",
@@ -294,13 +289,13 @@ class OpenRouterProvider(BaseProvider):
except RetryError as exc:
raise ProviderError(
"openrouter",
- f"All {_MAX_RETRIES} retries exhausted: {exc}",
+ f"All retries exhausted: {exc}",
) from exc
@retry(
- retry=retry_if_exception_type(ProviderError),
- stop=stop_after_attempt(_MAX_RETRIES),
- wait=wait_exponential_jitter(initial=_MIN_WAIT, max=_MAX_WAIT),
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
reraise=True,
)
async def _generate_with_retry(
@@ -325,7 +320,14 @@ class OpenRouterProvider(BaseProvider):
kwargs.update(_openrouter_reasoning_kwargs(reasoning))
response = await self._client.chat.completions.create(**kwargs)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("openrouter", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "openrouter",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("openrouter", str(exc), status_code=exc.status_code) from exc
@@ -378,7 +380,14 @@ class OpenRouterProvider(BaseProvider):
try:
stream = await self._client.chat.completions.create(**kwargs)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("openrouter", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "openrouter",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("openrouter", str(exc), status_code=exc.status_code) from exc
@@ -444,6 +453,13 @@ class OpenRouterProvider(BaseProvider):
stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
except _OpenAIRateLimitError as exc:
- raise RateLimitError("openrouter", str(exc), status_code=429) from exc
+ raise RateLimitError(
+ "openrouter",
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(
+ getattr(getattr(exc, "response", None), "headers", None)
+ ),
+ ) from exc
except _OpenAIAPIStatusError as exc:
raise ProviderError("openrouter", str(exc), status_code=exc.status_code) from exc
diff --git a/packages/core/src/repowise/core/rate_limiter.py b/packages/core/src/repowise/core/rate_limiter.py
index dfc5ba82a..7c88a1bf9 100644
--- a/packages/core/src/repowise/core/rate_limiter.py
+++ b/packages/core/src/repowise/core/rate_limiter.py
@@ -41,16 +41,31 @@ class RateLimitConfig:
# Default rate limit configs for known providers.
-# These are conservative defaults; operators can override via config.
+#
+# Philosophy: these are FLOOD GUARDS, not throttles. The provider's own 429
+# responses are the authoritative throttle signal, and every provider retries
+# them patiently (retry-after aware, up to 8 attempts — see providers/llm/
+# base.py). Throttling client-side below the provider's real limits just
+# leaves the API idle: the old openai default (150k TPM ÷ 20k reserved per
+# request) capped doc generation at 7.5 requests/min while Tier-1 already
+# allows 500 RPM. Low-tier accounts degrade gracefully via 429 backoff
+# instead of being pre-emptively slowed for everyone.
+#
+# Values sit near each provider's published upper tiers (2026):
+# - OpenAI Tier 4/5: 30k RPM, 40M+ TPM (mini/nano are 5-10x full models)
+# - Gemini paid Tier 1+: 300+ RPM, 1M+ TPM on flash
+# - Anthropic Tier 4: 4k RPM, 2M ITPM (cached reads don't count)
+# - DeepSeek: no published limits (dynamic, load-based)
+# - OpenRouter: no published RPM/TPM for paid models
PROVIDER_DEFAULTS: dict[str, RateLimitConfig] = {
- "anthropic": RateLimitConfig(requests_per_minute=50, tokens_per_minute=100_000),
- "openai": RateLimitConfig(requests_per_minute=60, tokens_per_minute=150_000),
- "openrouter": RateLimitConfig(requests_per_minute=60, tokens_per_minute=200_000),
- "gemini": RateLimitConfig(requests_per_minute=60, tokens_per_minute=1_000_000),
+ "anthropic": RateLimitConfig(requests_per_minute=2_000, tokens_per_minute=1_600_000),
+ "openai": RateLimitConfig(requests_per_minute=5_000, tokens_per_minute=4_000_000),
+ "openrouter": RateLimitConfig(requests_per_minute=1_000, tokens_per_minute=2_000_000),
+ "gemini": RateLimitConfig(requests_per_minute=1_000, tokens_per_minute=4_000_000),
# Ollama runs locally — effectively unlimited, but we cap to avoid OOM
"ollama": RateLimitConfig(requests_per_minute=1_000, tokens_per_minute=10_000_000),
- "litellm": RateLimitConfig(requests_per_minute=60, tokens_per_minute=150_000),
- "deepseek": RateLimitConfig(requests_per_minute=60, tokens_per_minute=200_000),
+ "litellm": RateLimitConfig(requests_per_minute=1_000, tokens_per_minute=2_000_000),
+ "deepseek": RateLimitConfig(requests_per_minute=1_000, tokens_per_minute=5_000_000),
}
diff --git a/tests/unit/test_providers/conftest.py b/tests/unit/test_providers/conftest.py
new file mode 100644
index 000000000..5c3da5a4e
--- /dev/null
+++ b/tests/unit/test_providers/conftest.py
@@ -0,0 +1,19 @@
+"""Shared fixtures for provider unit tests."""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.providers.llm import base as _llm_base
+
+
[email protected](autouse=True)
+def fast_retry_waits(monkeypatch):
+ """Collapse retry backoff waits so persistent-error tests stay fast.
+
+ The production 429 policy waits up to ~60s cumulatively (see
+ ``provider_retry_wait``). ``_WAIT_SCALE`` is read at call time, so
+ scaling it down keeps the retry *shape* (attempt counts, retryability)
+ under test while making the sleeps negligible.
+ """
+ monkeypatch.setattr(_llm_base, "_WAIT_SCALE", 0.001)