:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(anthropic): wait out a long retry-after on claude code report syncs (#119255)
posthog[bot]6b7088c671df
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.pyindex ce69085f16da..0b9d080014de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py@@ -48,11 +48,12 @@ ENTITY_PAGE_SIZE = 1000# a transient blip without letting a genuinely broken endpoint retry for long.MAX_RETRY_ATTEMPTS = 8# The usage/cost/analytics report endpoints share one organization-level rate limit on Anthropic's-# Admin API, which answers 429 with no `Retry-After`. The `anthropic-ratelimit-requests-reset`-# instant is often already stale when we read it, so the client falls back to exponential backoff.-# A per-minute limit needs a budget that outlasts the window, so give the report endpoints more-# attempts and a higher backoff ceiling than the entity lists — each wait still stays capped, so a-# sync never stalls indefinitely.+# Admin API. Some of them, such as the Claude Code usage report, answer 429 with a `Retry-After` of+# up to an hour, which is the time until the budget is full again. The budget refills continuously,+# so the report clients cap each server delay at the shared limit and wait it out, instead of+# failing the request at once. A 429 without a usable delay falls back to exponential backoff.+# Give the report endpoints more attempts and a higher backoff ceiling than the entity lists. Each+# wait stays capped, so a sync never stalls indefinitely.REPORT_MAX_RETRY_ATTEMPTS = 12REPORT_RETRY_BACKOFF_MAX_SECONDS = 300.0# The waits of the report attempts above add up to about 1,100 seconds, which is more than the@@ -852,6 +853,7 @@ def anthropic_source(if is_report_endpoint:client_config["retry_backoff_max_seconds"] = REPORT_RETRY_BACKOFF_MAX_SECONDSclient_config["retry_budget_seconds"] = REPORT_RETRY_BUDGET_SECONDS+ client_config["cap_retry_after"] = Trueresume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.pyindex 49e3d211216c..623bf837645f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py@@ -366,6 +366,20 @@ class TestClaudeCodeDayFanOut:assert result.params("page") == ["P2"]assert result.params("starting_at") == [today.isoformat()]+ def test_waits_out_a_rate_limit_that_asks_for_an_hour(self) -> None:+ today = datetime.now(UTC).date()+ result = _driver().run(+ "claude_code_analytics",+ [+ ScriptedResponse(status=429, headers={"Retry-After": "3600"}),+ _cc_page([_cc_record()], has_more=False, next_page=None),+ ],+ incremental_field="date",+ db_incremental_field_last_value=_midnight(today),+ )+ assert result.raised is None+ assert len(result.rows) == 1+class TestRetiredEndpoint:def test_a_dropped_endpoint_fails_non_retryably(self) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.pyindex e11fe8cf0e26..868d953677e4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py@@ -360,6 +360,7 @@ def create_resources(allow_redirects=client_config.get("allow_redirects", True),request_timeout=client_config.get("request_timeout"),capture=client_config.get("capture", True),+ cap_retry_after=client_config.get("cap_retry_after", False),**_retry_limit_overrides(client_config),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.pyindex b7f1f0287cad..5cb6aefb6d26 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py@@ -237,8 +237,7 @@ def _seconds_until_anthropic_replenish(value: str) -> Optional[float]:# the parser that turns its value into "seconds from now until the budget replenishes"._RATE_LIMIT_RESET_HEADERS: tuple[tuple[str, Callable[[str], Optional[float]]], ...] = (# Anthropic rate limits per organization and documents this RFC 3339 instant as when the request- # budget replenishes. Its Admin API (the usage/cost report endpoints an Anthropic source pages- # through) answers 429 without a ``Retry-After``, so this is the only delay it advertises.+ # budget replenishes. Some Admin API report endpoints send it on a 429 without a ``Retry-After``.("anthropic-ratelimit-requests-reset", _seconds_until_anthropic_replenish),# Sentry signals its rate-limit window with a UNIX epoch timestamp rather than ``Retry-After``,# and Sentry's flat / fan-out endpoints (e.g. ``project_users``) sync through this client too.@@ -290,11 +289,15 @@ def _parse_retry_after(response: Response) -> Optional[float]:return None+def _retry_after_limit(client: Any) -> float:+ limit = getattr(client, "_retry_after_max", None)+ return max_retry_after_seconds() if limit is None else limit++def _retry_after_exceeds_limit(client: Any, exc: Optional[BaseException]) -> bool:if not isinstance(exc, RESTClientRetryableError) or exc.retry_after is None:return False- limit = getattr(client, "_retry_after_max", None)- return exc.retry_after > (max_retry_after_seconds() if limit is None else limit)+ return exc.retry_after > _retry_after_limit(client)def _stop_after_client_attempts(state: RetryCallState) -> bool:@@ -381,6 +384,7 @@ class RESTClient:capture: bool = True,retry_budget_seconds: Optional[float] = None,retry_after_max_seconds: Optional[float] = None,+ cap_retry_after: bool = False,) -> None:self.base_url = base_url or ""self.headers = headers or {}@@ -394,6 +398,10 @@ class RESTClient:# does not fail the run.self._retry_budget = retry_budget_secondsself._retry_after_max = retry_after_max_seconds+ # When True, a server delay above the limit becomes a wait of the limit, and the request+ # does not fail at once. Use it for a vendor whose advertised delay is longer than the+ # time the client must wait before a retry can succeed.+ self._cap_retry_after = cap_retry_after# Per-request (connect, read) timeout in seconds handed to ``session.send``. Left None,# the session's own default applies, and after that the tracked adapter's# ``default_request_timeout()``. Sources talking to a customer-controlled host should set@@ -674,6 +682,8 @@ class RESTClient:# the percent-encoded form an `api_key` query credential takes in the URL, so scheme/# host/path-only is what keeps an encoded credential out of the persisted `latest_error`.retry_after = _parse_retry_after(response)+ if retry_after is not None and self._cap_retry_after:+ retry_after = min(retry_after, _retry_after_limit(self))message = f"HTTP {response.status_code} for {_safe_url(response.url)}"error = RESTClientRetryableError(self._redact(message), retry_after=retry_after)if _retry_after_exceeds_limit(self, error):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.pyindex df9983c64c31..c989a985ba2f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py@@ -221,6 +221,9 @@ class ClientConfig(TypedDict, total=False):# `DATA_WAREHOUSE_SOURCE_*` settings apply. Raise both for a vendor with a longer rate limit window.retry_budget_seconds: floatretry_after_max_seconds: float+ # When True, a server-provided delay above `retry_after_max_seconds` becomes a wait of that+ # limit, and the request does not fail at once. Left unset, such a request fails without a wait.+ cap_retry_after: bool# SSRF host-pinning. When set (even to an empty list), every outgoing request URL —# including paginator next-page links and seeded resume URLs — must resolve to one of# these hosts; the base_url host is always implicitly allowed. Off-host URLs are rejected