:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): retry byok 429s from first-party providers (#128370)
Christiaan Hendriksen18db57504894
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex f75364bfcfcd..a194ddd6e3b7 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -693,6 +693,9 @@ def call_llm_judge(provider_key=provider_key,config=config,capture_analytics=False,+ # Temporal retries a rate limit after the provider's delay. SDK retries underneath would+ # multiply the calls and can run past the activity's timeout before the last attempt ends.+ max_retries=0,)probability: float | None = None@@ -932,10 +935,12 @@ def call_llm_judge(# Decision endpoints raise this error for outages too (408, 503, 529). Only a 429 on the# customer's own key is a limit that the customer sets and PostHog cannot act on.customer_limited = is_byok and e.status == 429- if customer_limited and last and not backfill:- increment_user_errors("rate_limit_retries_exhausted", provider=provider)- return _build_rate_limit_skip_result(e, allows_na=allows_na, output_type=output_type)- increment_errors("rate_limit", provider=provider)+ if customer_limited:+ if last and not backfill:+ increment_user_errors("rate_limit_retries_exhausted", provider=provider)+ return _build_rate_limit_skip_result(e, allows_na=allows_na, output_type=output_type)+ else:+ increment_errors("rate_limit", provider=provider)# A retry usually gets through, so only an outage that outlasts every attempt reaches error tracking.error_class = ApplicationError if last and not customer_limited else NonReportableApplicationErrorraise error_class(diff --git a/posthog/temporal/ai_observability/run_tagger.py b/posthog/temporal/ai_observability/run_tagger.pyindex e5ef202ddd23..2a410211eae0 100644--- a/posthog/temporal/ai_observability/run_tagger.py+++ b/posthog/temporal/ai_observability/run_tagger.py@@ -314,6 +314,9 @@ def execute_tagger_activity(inputs: ExecuteTaggerInputs) -> dict[str, Any]:provider_key=provider_key,config=config,capture_analytics=False,+ # Temporal retries a rate limit after the provider's delay. SDK retries underneath would+ # multiply the calls and can run past the activity's timeout before the last attempt ends.+ max_retries=0,)try:diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex cf7463a7ff36..d22d5158566f 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -1089,6 +1089,7 @@ def test_custom_provider_rate_limit_retries_without_disabling_the_evaluation(],) as transport,patch("posthog.temporal.ai_observability.evaluation_llm_judge.increment_user_errors") as user_errors,+ patch("posthog.temporal.ai_observability.evaluation_llm_judge.increment_errors") as system_errors,):spec.return_value.resolve.return_value = MagicMock(provider=provider,@@ -1126,6 +1127,9 @@ def test_custom_provider_rate_limit_retries_without_disabling_the_evaluation(assert [call.args[0] for call in user_errors.call_args_list] == (["rate_limit_retries_exhausted"] if expected == "skipped" else [])+ assert [call.args[0] for call in system_errors.call_args_list] == (+ [] if is_byok and status == 429 else ["rate_limit"]+ )result = call_llm_judge(evaluation=evaluation,diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.pyindex 6c6ea8313dd5..4b448c19ddc1 100644--- a/products/ai_observability/backend/llm/client.py+++ b/products/ai_observability/backend/llm/client.py@@ -36,9 +36,11 @@ class Client:groups: dict[str, Any] | None = None,capture_analytics: bool = True,privacy_mode: bool = False,+ max_retries: int | None = None,):self.provider_key = provider_keyself.config = config+ self.max_retries = max_retriesself.analytics = AnalyticsContext(distinct_id=distinct_id,trace_id=trace_id or str(uuid.uuid4()),@@ -62,14 +64,14 @@ class Client:def complete(self, request: CompletionRequest) -> CompletionResponse:"""Non-streaming completion."""self._validate_provider(request.provider)- provider = _get_provider(request.provider, self.provider_key)+ provider = _get_provider(request.provider, self.provider_key, self.max_retries)api_key, base_url = self._resolve_credentials()return provider.complete(request, api_key, self.analytics, base_url)def stream(self, request: CompletionRequest) -> Generator[StreamChunk]:"""Streaming completion."""self._validate_provider(request.provider)- provider = _get_provider(request.provider, self.provider_key)+ provider = _get_provider(request.provider, self.provider_key, self.max_retries)api_key, base_url = self._resolve_credentials()yield from provider.stream(request, api_key, self.analytics, base_url)@@ -101,7 +103,20 @@ class Client:return _get_provider(provider).recommended_models()-def _get_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Provider":+def _get_provider(+ name: str, provider_key: "LLMProviderKey | None" = None, max_retries: int | None = None+) -> "Provider":+ from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter+ from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter++ provider = _build_provider(name, provider_key)+ # Gemini's SDK does not retry by default, so only these two need the override.+ if max_retries is not None and isinstance(provider, OpenAIAdapter | AnthropicAdapter):+ provider.max_retries = max_retries+ return provider+++def _build_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Provider":"""Get provider by name. Anthropic, Azure and OpenAI-compatible read extra config from provider_key."""from typing import castdiff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.pyindex 115b302767e8..cbdab255191b 100644--- a/products/ai_observability/backend/llm/providers/anthropic.py+++ b/products/ai_observability/backend/llm/providers/anthropic.py@@ -28,7 +28,7 @@ from products.ai_observability.backend.llm.errors import (ProviderConfigurationError,ProviderConnectionError,QuotaExceededError,- RateLimitError,+ RetryableRateLimitError,StructuredOutputParseError,error_field_for_message,is_context_window_error_message,@@ -197,6 +197,8 @@ class AnthropicAdapter:name = "anthropic"+ max_retries: int = anthropic.DEFAULT_MAX_RETRIES+def __init__(self, foundry_endpoint: str = ""):# Set when the key belongs to a Microsoft Foundry resource that serves Claude.self.foundry_endpoint = foundry_endpoint@@ -208,6 +210,7 @@ class AnthropicAdapter:kwargs: dict[str, Any] = {"api_key": api_key,"timeout": AnthropicConfig.TIMEOUT,+ "max_retries": self.max_retries,"http_client": tagged_http_client(timeout=AnthropicConfig.TIMEOUT),}if self.foundry_endpoint:@@ -334,7 +337,7 @@ class AnthropicAdapter:if isinstance(error, anthropic.RateLimitError):if _is_quota_or_billing_error(error):return QuotaExceededError(str(error))- return RateLimitError(str(error))+ return RetryableRateLimitError(str(error), error.response.headers.get("Retry-After"))if isinstance(error, anthropic.APIConnectionError):# Transient transport failure (connection reset, read timeout). Map to a quiet# retryable error so the caller retries silently instead of spamming error tracking.diff --git a/products/ai_observability/backend/llm/providers/gemini.py b/products/ai_observability/backend/llm/providers/gemini.pyindex 6a55e65a08f9..60d060097626 100644--- a/products/ai_observability/backend/llm/providers/gemini.py+++ b/products/ai_observability/backend/llm/providers/gemini.py@@ -24,7 +24,7 @@ from products.ai_observability.backend.llm.errors import (ProviderConnectionError,ProviderRequestRejectedError,QuotaExceededError,- RateLimitError,+ RetryableRateLimitError,StructuredOutputParseError,stream_error_chunk,)@@ -44,6 +44,16 @@ from products.ai_observability.backend.providers.formatters.gemini_formatter implogger = logging.getLogger(__name__)+def _retry_delay(error: APIError) -> str | None:+ """The seconds in the error's google.rpc.RetryInfo detail, such as "27" from "27s"."""+ body = error.details.get("error", {}) if isinstance(error.details, dict) else {}+ for detail in body.get("details") or []:+ if isinstance(detail, dict) and str(detail.get("@type", "")).endswith("google.rpc.RetryInfo"):+ delay = str(detail.get("retryDelay", ""))+ return delay.removesuffix("s") or None+ return None++class GeminiConfig:TEMPERATURE: float = 0# Timeout in milliseconds for API calls. Set high to accommodate slow reasoning models.@@ -167,9 +177,12 @@ class GeminiAdapter:if status_code == 403 or "permission denied" in error_message:return ModelPermissionError(model)if status_code == 429 or "rate limit" in error_message or "resource exhausted" in error_message:- if "quota" in error_message or "billing" in error_message:+ # Per-minute limits also say "quota". Google adds a retry delay when waiting+ # helps, so its presence separates a rate limit from exhausted billing.+ retry_delay = _retry_delay(error)+ if retry_delay is None and ("quota" in error_message or "billing" in error_message):return QuotaExceededError(str(error))- return RateLimitError(str(error))+ return RetryableRateLimitError(str(error), retry_delay)# Google returns a 404-class error (often with "no longer available") when a# model is retired/deprecated. Map it so call_llm_judge disables the eval# gracefully instead of burning Temporal retries on an unhandled exception.diff --git a/products/ai_observability/backend/llm/providers/openai.py b/products/ai_observability/backend/llm/providers/openai.pyindex d19ee107da8c..86f1ccdc1ffd 100644--- a/products/ai_observability/backend/llm/providers/openai.py+++ b/products/ai_observability/backend/llm/providers/openai.py@@ -31,7 +31,7 @@ from products.ai_observability.backend.llm.errors import (OutputTokenLimitError,ProviderConnectionError,QuotaExceededError,- RateLimitError,+ RetryableRateLimitError,StructuredOutputParseError,is_context_window_error_message,is_output_limit_error_message,@@ -272,7 +272,7 @@ class OpenAIAdapter:error_code = error_body.get("code", "") or error_body.get("error", {}).get("code", "")if error_code == "insufficient_quota":return QuotaExceededError(str(error))- return RateLimitError(str(error))+ return RetryableRateLimitError(str(error), error.response.headers.get("Retry-After"))if isinstance(error, openai.APIConnectionError):# Transient transport failure (connection reset, read timeout). Map to a quiet# retryable error so the caller retries silently instead of spamming error tracking.diff --git a/products/ai_observability/backend/llm/providers/openai_compatible.py b/products/ai_observability/backend/llm/providers/openai_compatible.pyindex 040b79d76ae9..52e4c72cc106 100644--- a/products/ai_observability/backend/llm/providers/openai_compatible.py+++ b/products/ai_observability/backend/llm/providers/openai_compatible.py@@ -32,8 +32,6 @@ from products.ai_observability.backend.llm.errors import (ProviderHostUnresolvedError,ProviderRequestRejectedError,ProviderTimeoutError,- RateLimitError,- RetryableRateLimitError,error_field_for_message,)from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client@@ -170,10 +168,7 @@ class OpenAICompatibleAdapter(OpenAIAdapter):return ProviderRequestRejectedError(RESPONSE_LIMIT_MESSAGE)if isinstance(cause, httpx.TimeoutException):return ProviderTimeoutError(self.request_timeout)- mapped = super()._mapped_error(error, model)- if isinstance(error, openai.RateLimitError) and isinstance(mapped, RateLimitError):- return RetryableRateLimitError(str(error), error.response.headers.get("Retry-After"))- return mapped+ return super()._mapped_error(error, model)def complete(self,diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.pyindex 3a5fd718195f..22687db12193 100644--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py@@ -12,6 +12,7 @@ from products.ai_observability.backend.llm.errors import (ModelPermissionError,OutputTokenLimitError,ProviderConfigurationError,+ RetryableRateLimitError,)from products.ai_observability.backend.llm.providers.anthropic import (AnthropicAdapter,@@ -350,6 +351,31 @@ class TestAnthropicErrorMapping:analytics=AnalyticsContext(capture=False),)+ def test_rate_limit_429_is_retryable_after_the_providers_delay(self):+ request = httpx.Request("POST", "https://api.anthropic.com/v1/messages")+ with patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic") as mock_cls:+ mock_client = MagicMock()+ mock_cls.return_value = mock_client+ mock_client.messages.create.side_effect = anthropic.RateLimitError(+ "Rate limited",+ response=httpx.Response(status_code=429, request=request, headers={"retry-after": "20"}),+ body={"type": "error", "error": {"type": "rate_limit_error", "message": "Rate limited"}},+ )++ with pytest.raises(RetryableRateLimitError) as error:+ AnthropicAdapter().complete(+ CompletionRequest(+ model="claude-haiku-4-5",+ messages=[{"role": "user", "content": "hi"}],+ provider="anthropic",+ system="s",+ ),+ api_key="sk-ant-test",+ analytics=AnalyticsContext(capture=False),+ )++ assert error.value.retry_after == 20+class TestAnthropicStreamErrorSurfacing:def test_model_404_yields_actionable_message_instead_of_discarding_the_reason(self):diff --git a/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py b/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.pyindex ce10399fd8a6..8df3dfb16db3 100644--- a/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py+++ b/products/ai_observability/backend/llm/providers/test/test_gemini_adapter.py@@ -10,7 +10,8 @@ from products.ai_observability.backend.llm.errors import (ModelNotFoundError,ModelPermissionError,ProviderConnectionError,- RateLimitError,+ QuotaExceededError,+ RetryableRateLimitError,)from products.ai_observability.backend.llm.providers.gemini import GeminiAdapter, GeminiConfigfrom products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema@@ -39,7 +40,14 @@ class TestGeminiAdapterErrorMapping:),("403_permission", 403, "PERMISSION_DENIED", "permission denied", ModelPermissionError),("401_auth", 401, "UNAUTHENTICATED", "invalid api key", AuthenticationError),- ("429_rate_limit", 429, "RESOURCE_EXHAUSTED", "rate limit exceeded", RateLimitError),+ ("429_rate_limit", 429, "RESOURCE_EXHAUSTED", "rate limit exceeded", RetryableRateLimitError),+ (+ "429_billing_without_retry_delay",+ 429,+ "RESOURCE_EXHAUSTED",+ "Your project is out of credits. Check your billing details.",+ QuotaExceededError,+ ),# A cancelled call is retryable, so it shares the transport lane. Without this the# per-occurrence message files a new error tracking issue on every cancellation.("499_cancelled", 499, "CANCELLED", "The operation was cancelled.", ProviderConnectionError),@@ -68,6 +76,33 @@ class TestGeminiAdapterErrorMapping:with pytest.raises(expected):adapter.complete(request, api_key="test-key", analytics=AnalyticsContext(capture=False))+ def test_per_minute_quota_429_is_retryable_after_the_providers_delay(self):+ error = ClientError(+ 429,+ {+ "error": {+ "code": 429,+ "status": "RESOURCE_EXHAUSTED",+ "message": "You exceeded your current quota. Quota exceeded for metric: requests per minute.",+ "details": [+ {"@type": "type.googleapis.com/google.rpc.QuotaFailure", "violations": []},+ {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "27s"},+ ],+ }+ },+ )+ mock_client = MagicMock()+ mock_client.models.generate_content.side_effect = error+ request = CompletionRequest(+ model="gemini-2.0-flash-lite", system="s", messages=[{"role": "user", "content": "hi"}], provider="gemini"+ )++ with patch("products.ai_observability.backend.llm.providers.gemini.genai.Client", return_value=mock_client):+ with pytest.raises(RetryableRateLimitError) as raised:+ GeminiAdapter().complete(request, api_key="test-key", analytics=AnalyticsContext(capture=False))++ assert raised.value.retry_after == 27+def test_transport_error_is_mapped_to_provider_connection_error(self):# google-genai leaks raw httpx transport errors (connection reset) rather than wrapping# them in APIError, so they'd escape the mapping above without a dedicated branch.diff --git a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.pyindex 6b28aa0fcfa2..9f1e6f3dd776 100644--- a/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py+++ b/products/ai_observability/backend/llm/providers/test/test_openai_adapter.py@@ -18,8 +18,10 @@ from products.ai_observability.backend.llm.errors import (ContextWindowExceededError,OutputTokenLimitError,QuotaExceededError,+ RetryableRateLimitError,StructuredOutputParseError,)+from products.ai_observability.backend.llm.providers.azure_openai import AzureOpenAIAdapterfrom products.ai_observability.backend.llm.providers.openai import OpenAIAdapter, OpenAIConfigfrom products.ai_observability.backend.llm.providers.openrouter import OpenRouterAdapterfrom products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema@@ -304,6 +306,37 @@ class TestOpenAIAdapterErrorMapping:request_no_structured_output, api_key="sk-test", analytics=AnalyticsContext(capture=False))+ @parameterized.expand(+ [+ ("openai", OpenAIAdapter(), "products.ai_observability.backend.llm.providers.openai.openai.OpenAI"),+ (+ "azure_openai",+ AzureOpenAIAdapter(azure_endpoint="https://example-resource.openai.azure.com/"),+ "products.ai_observability.backend.llm.providers.azure_openai.openai.AzureOpenAI",+ ),+ ]+ )+ def test_429_is_retryable_after_the_providers_delay(self, provider: str, adapter: OpenAIAdapter, client_path: str):+ mock_client = MagicMock()+ mock_client.chat.completions.create.side_effect = openai.RateLimitError(+ "Too many requests",+ response=httpx.Response(+ status_code=429,+ request=httpx.Request("POST", "https://example.invalid/v1/chat/completions"),+ headers={"Retry-After": "20"},+ ),+ body={"error": {"message": "Too many requests", "code": "rate_limit_exceeded"}},+ )+ request = CompletionRequest(+ model="gpt-4.1", system="s", messages=[{"role": "user", "content": "hi"}], provider=provider+ )++ with patch(client_path, return_value=mock_client):+ with pytest.raises(RetryableRateLimitError) as error:+ adapter.complete(request, api_key="sk-test", analytics=AnalyticsContext(capture=False))++ assert error.value.retry_after == 20+@parameterized.expand([("server_error", 500), ("precondition_failed", 412)])def test_non_quota_status_error_is_not_swallowed(self, _name: str, status_code: int):adapter = OpenAIAdapter()diff --git a/products/ai_observability/backend/llm/test/test_client.py b/products/ai_observability/backend/llm/test/test_client.pyindex 05e7697b472b..3abeaa8f9278 100644--- a/products/ai_observability/backend/llm/test/test_client.py+++ b/products/ai_observability/backend/llm/test/test_client.py@@ -1,13 +1,18 @@import pytest-from unittest.mock import MagicMock+from unittest.mock import MagicMock, patchfrom django.test import SimpleTestCase+import httpxfrom parameterized import parameterizedfrom products.ai_observability.backend.llm.client import Clientfrom products.ai_observability.backend.llm.config import ProviderConfig-from products.ai_observability.backend.llm.errors import ProviderMismatchError, UnsupportedProviderError+from products.ai_observability.backend.llm.errors import (+ ProviderMismatchError,+ RetryableRateLimitError,+ UnsupportedProviderError,+)from products.ai_observability.backend.llm.types import CompletionRequest@@ -217,3 +222,27 @@ class TestEffectiveCredentials(SimpleTestCase):assert api_key == "config-key"assert base_url is None+++class TestSdkRetries(SimpleTestCase):+ @parameterized.expand([("openai", "gpt-4.1"), ("anthropic", "claude-haiku-4-5")])+ def test_max_retries_zero_makes_one_call_on_a_429(self, provider: str, model: str):+ calls: list[httpx.Request] = []++ def handler(request: httpx.Request) -> httpx.Response:+ calls.append(request)+ return httpx.Response(429, json={"error": {"type": "rate_limit_error", "message": "Too many requests"}})++ client = Client(config=ProviderConfig(api_key="sk-test"), capture_analytics=False, max_retries=0)+ request = CompletionRequest(+ model=model, system="s", messages=[{"role": "user", "content": "hi"}], provider=provider+ )++ with patch(+ "products.ai_observability.backend.llm.providers._diagnostics.tagged_http_client",+ return_value=httpx.Client(transport=httpx.MockTransport(handler)),+ ):+ with pytest.raises(RetryableRateLimitError):+ client.complete(request)++ assert len(calls) == 1