Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): retry byok 429s from first-party providers (#128370)

Christiaan Hendriksen18db57504894
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index f75364bfcfcd..a194ddd6e3b7 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -693,6 +693,9 @@ def call_llm_judge(
provider_key=provider_key,
config=config,
capture_analytics=False,
+ # Temporal retries a rate limit after the provider's delay. SDK retries underneath would
+ # multiply the calls and can run past the activity's timeout before the last attempt ends.
+ max_retries=0,
)
probability: float | None = None
@@ -932,10 +935,12 @@ def call_llm_judge(
# Decision endpoints raise this error for outages too (408, 503, 529). Only a 429 on the
# customer's own key is a limit that the customer sets and PostHog cannot act on.
customer_limited = is_byok and e.status == 429
- if customer_limited and last and not backfill:
- increment_user_errors("rate_limit_retries_exhausted", provider=provider)
- return _build_rate_limit_skip_result(e, allows_na=allows_na, output_type=output_type)
- increment_errors("rate_limit", provider=provider)
+ if customer_limited:
+ if last and not backfill:
+ increment_user_errors("rate_limit_retries_exhausted", provider=provider)
+ return _build_rate_limit_skip_result(e, allows_na=allows_na, output_type=output_type)
+ else:
+ increment_errors("rate_limit", provider=provider)
# A retry usually gets through, so only an outage that outlasts every attempt reaches error tracking.
error_class = ApplicationError if last and not customer_limited else NonReportableApplicationError
raise error_class(
diff --git a/posthog/temporal/ai_observability/run_tagger.py b/posthog/temporal/ai_observability/run_tagger.py
index e5ef202ddd23..2a410211eae0 100644
--- a/posthog/temporal/ai_observability/run_tagger.py
+++ b/posthog/temporal/ai_observability/run_tagger.py
@@ -314,6 +314,9 @@ def execute_tagger_activity(inputs: ExecuteTaggerInputs) -> dict[str, Any]:
provider_key=provider_key,
config=config,
capture_analytics=False,
+ # Temporal retries a rate limit after the provider's delay. SDK retries underneath would
+ # multiply the calls and can run past the activity's timeout before the last attempt ends.
+ max_retries=0,
)
try:
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index cf7463a7ff36..d22d5158566f 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -1089,6 +1089,7 @@ def test_custom_provider_rate_limit_retries_without_disabling_the_evaluation(
],
) as transport,
patch("posthog.temporal.ai_observability.evaluation_llm_judge.increment_user_errors") as user_errors,
+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.increment_errors") as system_errors,
):
spec.return_value.resolve.return_value = MagicMock(
provider=provider,
@@ -1126,6 +1127,9 @@ def test_custom_provider_rate_limit_retries_without_disabling_the_evaluation(
assert [call.args[0] for call in user_errors.call_args_list] == (
["rate_limit_retries_exhausted"] if expected == "skipped" else []
)
+ assert [call.args[0] for call in system_errors.call_args_list] == (
+ [] if is_byok and status == 429 else ["rate_limit"]
+ )
result = call_llm_judge(
evaluation=evaluation,
diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.py
index 6c6ea8313dd5..4b448c19ddc1 100644
--- a/products/ai_observability/backend/llm/client.py
+++ b/products/ai_observability/backend/llm/client.py
@@ -36,9 +36,11 @@ class Client:
groups: dict[str, Any] | None = None,
capture_analytics: bool = True,
privacy_mode: bool = False,
+ max_retries: int | None = None,
):
self.provider_key = provider_key
self.config = config
+ self.max_retries = max_retries
self.analytics = AnalyticsContext(
distinct_id=distinct_id,
trace_id=trace_id or str(uuid.uuid4()),
@@ -62,14 +64,14 @@ class Client:
def complete(self, request: CompletionRequest) -> CompletionResponse:
"""Non-streaming completion."""
self._validate_provider(request.provider)
- provider = _get_provider(request.provider, self.provider_key)
+ provider = _get_provider(request.provider, self.provider_key, self.max_retries)
api_key, base_url = self._resolve_credentials()
return provider.complete(request, api_key, self.analytics, base_url)
def stream(self, request: CompletionRequest) -> Generator[StreamChunk]:
"""Streaming completion."""
self._validate_provider(request.provider)
- provider = _get_provider(request.provider, self.provider_key)
+ provider = _get_provider(request.provider, self.provider_key, self.max_retries)
api_key, base_url = self._resolve_credentials()
yield from provider.stream(request, api_key, self.analytics, base_url)
@@ -101,7 +103,20 @@ class Client:
return _get_provider(provider).recommended_models()
-def _get_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Provider":
+def _get_provider(
+ name: str, provider_key: "LLMProviderKey | None" = None, max_retries: int | None = None
+) -> "Provider":
+ from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter
+ from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter
+
+ provider = _build_provider(name, provider_key)
+ # Gemini's SDK does not retry by default, so only these two need the override.
+ if max_retries is not None and isinstance(provider, OpenAIAdapter | AnthropicAdapter):
+ provider.max_retries = max_retries
+ return provider
+
+
+def _build_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Provider":
"""Get provider by name. Anthropic, Azure and OpenAI-compatible read extra config from provider_key."""
from typing import cast
diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.py
index 115b302767e8..cbdab255191b 100644
--- a/products/ai_observability/backend/llm/providers/anthropic.py
+++ b/products/ai_observability/backend/llm/providers/anthropic.py
@@ -28,7 +28,7 @@ from products.ai_observability.backend.llm.errors import (
ProviderConfigurationError,
ProviderConnectionError,
QuotaExceededError,
- RateLimitError,
+ RetryableRateLimitError,
StructuredOutputParseError,
error_field_for_message,
is_context_window_error_message,
@@ -197,6 +197,8 @@ class AnthropicAdapter:
name = "anthropic"
+ max_retries: int = anthropic.DEFAULT_MAX_RETRIES
+
def __init__(self, foundry_endpoint: str = ""):
# Set when the key belongs to a Microsoft Foundry resource that serves Claude.
self.foundry_endpoint = foundry_endpoint
@@ -208,6 +210,7 @@ class AnthropicAdapter:
kwargs: dict[str, Any] = {
"api_key": api_key,
"timeout": AnthropicConfig.TIMEOUT,
+ "max_retries": self.max_retries,
"http_client": tagged_http_client(timeout=AnthropicConfig.TIMEOUT),
}
if self.foundry_endpoint:
@@ -334,7 +337,7 @@ class AnthropicAdapter:
if isinstance(error, anthropic.RateLimitError):
if _is_quota_or_billing_error(error):
return QuotaExceededError(str(error))
- return RateLimitError(str(error))
+ return RetryableRateLimitError(str(error), error.response.headers.get("Retry-After"))
if isinstance(error, anthropic.APIConnectionError):
# Transient transport failure (connection reset, read timeout). Map to a quiet
# retryable error so the caller retries silently instead of spamming error tracking.
diff --git a/products/ai_observability/backend/llm/providers/gemini.py b/products/ai_observability/backend/llm/providers/gemini.py
index 6a55e65a08f9..60d060097626 100644
--- a/products/ai_observability/backend/llm/providers/gemini.py
+++ b/products/ai_observability/backend/llm/providers/gemini.py
@@ -24,7 +24,7 @@ from products.ai_observability.backend.llm.errors import (
ProviderConnectionError,
ProviderRequestRejectedError,
QuotaExceededError,
- RateLimitError,
+ RetryableRateLimitError,
StructuredOutputParseError,
stream_error_chunk,
)
@@ -44,6 +44,16 @@ from products.ai_observability.backend.providers.formatters.gemini_formatter imp
logger = logging.getLogger(__name__)
+def _retry_delay(error: APIError) -> str | None:
+ """The seconds in the error's google.rpc.RetryInfo detail, such as "27" from "27s"."""
+ body = error.details.get("error", {}) if isinstance(error.details, dict) else {}
+ for detail in body.get("details") or []:
+ if isinstance(detail, dict) and str(detail.get("@type", "")).endswith("google.rpc.RetryInfo"):
+ delay = str(detail.get("retryDelay", ""))
+ return delay.removesuffix("s") or None
+ return None
+
+
class GeminiConfig:
TEMPERATURE: float = 0
# Timeout in milliseconds for API calls. Set high to accommodate slow reasoning models.
@@ -167,9 +177,12 @@ class GeminiAdapter:
if status_code == 403 or "permission denied" in error_message:
return ModelPermissionError(model)
if status_code == 429 or "rate limit" in error_message or "resource exhausted" in error_message:
- if "quota" in error_message or "billing" in error_message:
+ # Per-minute limits also say "quota". Google adds a retry delay when waiting
+ # helps, so its presence separates a rate limit from exhausted billing.
+ retry_delay = _retry_delay(error)
+ if retry_delay is None and ("quota" in error_message or "billing" in error_message):
return QuotaExceededError(str(error))
- return RateLimitError(str(error))
+ return RetryableRateLimitError(str(error), retry_delay)
# Google returns a 404-class error (often with "no longer available") when a
# model is retired/deprecated. Map it so call_llm_judge disables the eval
# gracefully instead of burning Temporal retries on an unhandled exception.
diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.py
index d19ee107da8c..86f1ccdc1ffd 100644
--- a/products/ai_observability/backend/llm/providers/openai.py
+++ b/products/ai_observability/backend/llm/providers/openai.py
@@ -31,7 +31,7 @@ from products.ai_observability.backend.llm.errors import (
OutputTokenLimitError,
ProviderConnectionError,
QuotaExceededError,
- RateLimitError,
+ RetryableRateLimitError,
StructuredOutputParseError,
is_context_window_error_message,
is_output_limit_error_message,
@@ -272,7 +272,7 @@ class OpenAIAdapter:
error_code = error_body.get("code", "") or error_body.get("error", {}).get("code", "")
if error_code == "insufficient_quota":
return QuotaExceededError(str(error))
- return RateLimitError(str(error))
+ return RetryableRateLimitError(str(error), error.response.headers.get("Retry-After"))
if isinstance(error, openai.APIConnectionError):
# Transient transport failure (connection reset, read timeout). Map to a quiet
# retryable error so the caller retries silently instead of spamming error tracking.
diff --git a/products/ai_observability/backend/llm/providers/openai_compatible.py b/products/ai_observability/backend/llm/providers/openai_compatible.py
index 040b79d76ae9..52e4c72cc106 100644
--- a/products/ai_observability/backend/llm/providers/openai_compatible.py
+++ b/products/ai_observability/backend/llm/providers/openai_compatible.py
@@ -32,8 +32,6 @@ from products.ai_observability.backend.llm.errors import (
ProviderHostUnresolvedError,
ProviderRequestRejectedError,
ProviderTimeoutError,
- RateLimitError,
- RetryableRateLimitError,
error_field_for_message,
)
from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client
@@ -170,10 +168,7 @@ class OpenAICompatibleAdapter(OpenAIAdapter):
return ProviderRequestRejectedError(RESPONSE_LIMIT_MESSAGE)
if isinstance(cause, httpx.TimeoutException):
return ProviderTimeoutError(self.request_timeout)
- mapped = super()._mapped_error(error, model)
- if isinstance(error, openai.RateLimitError) and isinstance(mapped, RateLimitError):
- return RetryableRateLimitError(str(error), error.response.headers.get("Retry-After"))
- return mapped
+ return super()._mapped_error(error, model)
def complete(
self,
diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
index 3a5fd718195f..22687db12193 100644
--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py
+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
@@ -12,6 +12,7 @@ from products.ai_observability.backend.llm.errors import (
ModelPermissionError,
OutputTokenLimitError,
ProviderConfigurationError,
+ RetryableRateLimitError,
)
from products.ai_observability.backend.llm.providers.anthropic import (
AnthropicAdapter,
@@ -350,6 +351,31 @@ class TestAnthropicErrorMapping:
analytics=AnalyticsContext(capture=False),
)
+ def test_rate_limit_429_is_retryable_after_the_providers_delay(self):
+ request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")
+ with patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic") as mock_cls:
+ mock_client = MagicMock()
+ mock_cls.return_value = mock_client
+ mock_client.messages.create.side_effect = anthropic.RateLimitError(
+ "Rate limited",
+ response=httpx.Response(status_code=429, request=request, headers={"retry-after": "20"}),
+ body={"type": "error", "error": {"type": "rate_limit_error", "message": "Rate limited"}},
+ )
+
+ with pytest.raises(RetryableRateLimitError) as error:
+ AnthropicAdapter().complete(
+ CompletionRequest(
+ model="claude-haiku-4-5",
+ messages=[{"role": "user", "content": "hi"}],
+ provider="anthropic",
+ system="s",
+ ),
+ api_key="sk-ant-test",
+ analytics=AnalyticsContext(capture=False),
+ )
+
+ assert error.value.retry_after == 20
+
class TestAnthropicStreamErrorSurfacing:
def test_model_404_yields_actionable_message_instead_of_discarding_the_reason(self):
diff --git a/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py b/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py
index ce10399fd8a6..8df3dfb16db3 100644
--- a/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py
+++ b/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py
@@ -10,7 +10,8 @@ from products.ai_observability.backend.llm.errors import (
ModelNotFoundError,
ModelPermissionError,
ProviderConnectionError,
- RateLimitError,
+ QuotaExceededError,
+ RetryableRateLimitError,
)
from products.ai_observability.backend.llm.providers.gemini import GeminiAdapter, GeminiConfig
from products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema
@@ -39,7 +40,14 @@ class TestGeminiAdapterErrorMapping:
),
("403_permission", 403, "PERMISSION_DENIED", "permission denied", ModelPermissionError),
("401_auth", 401, "UNAUTHENTICATED", "invalid api key", AuthenticationError),
- ("429_rate_limit", 429, "RESOURCE_EXHAUSTED", "rate limit exceeded", RateLimitError),
+ ("429_rate_limit", 429, "RESOURCE_EXHAUSTED", "rate limit exceeded", RetryableRateLimitError),
+ (
+ "429_billing_without_retry_delay",
+ 429,
+ "RESOURCE_EXHAUSTED",
+ "Your project is out of credits. Check your billing details.",
+ QuotaExceededError,
+ ),
# A cancelled call is retryable, so it shares the transport lane. Without this the
# per-occurrence message files a new error tracking issue on every cancellation.
("499_cancelled", 499, "CANCELLED", "The operation was cancelled.", ProviderConnectionError),
@@ -68,6 +76,33 @@ class TestGeminiAdapterErrorMapping:
with pytest.raises(expected):
adapter.complete(request, api_key="test-key", analytics=AnalyticsContext(capture=False))
+ def test_per_minute_quota_429_is_retryable_after_the_providers_delay(self):
+ error = ClientError(
+ 429,
+ {
+ "error": {
+ "code": 429,
+ "status": "RESOURCE_EXHAUSTED",
+ "message": "You exceeded your current quota. Quota exceeded for metric: requests per minute.",
+ "details": [
+ {"@type": "type.googleapis.com/google.rpc.QuotaFailure", "violations": []},
+ {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "27s"},
+ ],
+ }
+ },
+ )
+ mock_client = MagicMock()
+ mock_client.models.generate_content.side_effect = error
+ request = CompletionRequest(
+ model="gemini-2.0-flash-lite", system="s", messages=[{"role": "user", "content": "hi"}], provider="gemini"
+ )
+
+ with patch("products.ai_observability.backend.llm.providers.gemini.genai.Client", return_value=mock_client):
+ with pytest.raises(RetryableRateLimitError) as raised:
+ GeminiAdapter().complete(request, api_key="test-key", analytics=AnalyticsContext(capture=False))
+
+ assert raised.value.retry_after == 27
+
def test_transport_error_is_mapped_to_provider_connection_error(self):
# google-genai leaks raw httpx transport errors (connection reset) rather than wrapping
# them in APIError, so they'd escape the mapping above without a dedicated branch.
diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
index 6b28aa0fcfa2..9f1e6f3dd776 100644
--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py
@@ -18,8 +18,10 @@ from products.ai_observability.backend.llm.errors import (
ContextWindowExceededError,
OutputTokenLimitError,
QuotaExceededError,
+ RetryableRateLimitError,
StructuredOutputParseError,
)
+from products.ai_observability.backend.llm.providers.azure_openai import AzureOpenAIAdapter
from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfig
from products.ai_observability.backend.llm.providers.openrouter import OpenRouterAdapter
from products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema
@@ -304,6 +306,37 @@ class TestOpenAIAdapterErrorMapping:
request_no_structured_output, api_key="sk-test", analytics=AnalyticsContext(capture=False)
)
+ @parameterized.expand(
+ [
+ ("openai", OpenAIAdapter(), "products.ai_observability.backend.llm.providers.openai.openai.OpenAI"),
+ (
+ "azure_openai",
+ AzureOpenAIAdapter(azure_endpoint="https://example-resource.openai.azure.com/"),
+ "products.ai_observability.backend.llm.providers.azure_openai.openai.AzureOpenAI",
+ ),
+ ]
+ )
+ def test_429_is_retryable_after_the_providers_delay(self, provider: str, adapter: OpenAIAdapter, client_path: str):
+ mock_client = MagicMock()
+ mock_client.chat.completions.create.side_effect = openai.RateLimitError(
+ "Too many requests",
+ response=httpx.Response(
+ status_code=429,
+ request=httpx.Request("POST", "https://example.invalid/v1/chat/completions"),
+ headers={"Retry-After": "20"},
+ ),
+ body={"error": {"message": "Too many requests", "code": "rate_limit_exceeded"}},
+ )
+ request = CompletionRequest(
+ model="gpt-4.1", system="s", messages=[{"role": "user", "content": "hi"}], provider=provider
+ )
+
+ with patch(client_path, return_value=mock_client):
+ with pytest.raises(RetryableRateLimitError) as error:
+ adapter.complete(request, api_key="sk-test", analytics=AnalyticsContext(capture=False))
+
+ assert error.value.retry_after == 20
+
@parameterized.expand([("server_error", 500), ("precondition_failed", 412)])
def test_non_quota_status_error_is_not_swallowed(self, _name: str, status_code: int):
adapter = OpenAIAdapter()
diff --git a/products/ai_observability/backend/llm/test/test_client.py b/products/ai_observability/backend/llm/test/test_client.py
index 05e7697b472b..3abeaa8f9278 100644
--- a/products/ai_observability/backend/llm/test/test_client.py
+++ b/products/ai_observability/backend/llm/test/test_client.py
@@ -1,13 +1,18 @@
import pytest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
from django.test import SimpleTestCase
+import httpx
from parameterized import parameterized
from products.ai_observability.backend.llm.client import Client
from products.ai_observability.backend.llm.config import ProviderConfig
-from products.ai_observability.backend.llm.errors import ProviderMismatchError, UnsupportedProviderError
+from products.ai_observability.backend.llm.errors import (
+ ProviderMismatchError,
+ RetryableRateLimitError,
+ UnsupportedProviderError,
+)
from products.ai_observability.backend.llm.types import CompletionRequest
@@ -217,3 +222,27 @@ class TestEffectiveCredentials(SimpleTestCase):
assert api_key == "config-key"
assert base_url is None
+
+
+class TestSdkRetries(SimpleTestCase):
+ @parameterized.expand([("openai", "gpt-4.1"), ("anthropic", "claude-haiku-4-5")])
+ def test_max_retries_zero_makes_one_call_on_a_429(self, provider: str, model: str):
+ calls: list[httpx.Request] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request)
+ return httpx.Response(429, json={"error": {"type": "rate_limit_error", "message": "Too many requests"}})
+
+ client = Client(config=ProviderConfig(api_key="sk-test"), capture_analytics=False, max_retries=0)
+ request = CompletionRequest(
+ model=model, system="s", messages=[{"role": "user", "content": "hi"}], provider=provider
+ )
+
+ with patch(
+ "products.ai_observability.backend.llm.providers._diagnostics.tagged_http_client",
+ return_value=httpx.Client(transport=httpx.MockTransport(handler)),
+ ):
+ with pytest.raises(RetryableRateLimitError):
+ client.complete(request)
+
+ assert len(calls) == 1